diff --git a/README.md b/README.md index b7ebfff9d..b9aa9e2ca 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status][travis-img]][travis] +[![Build Status]][ghactions] [![Code Climate][codeclimate-img]][codeclimate] [![Codacy Grade][codacy-grade-img]][codacy-grade] [![Codacy Coverage][codacy-coverage-img]][codacy-coverage] @@ -59,8 +59,8 @@ See the [manual][docs] for more usage info. [FAQ]: http://alot.readthedocs.io/en/latest/faq.html [features]: https://github.com/pazz/alot/issues?labels=feature -[travis]: https://travis-ci.com/pazz/alot -[travis-img]: https://travis-ci.com/pazz/alot.svg?branch=master +[Build Status]: https://github.com/pazz/alot/actions/workflows/check.yml/badge.svg +[ghactions]: https://github.com/pazz/alot/actions [codacy-coverage]: https://www.codacy.com/app/patricktotzke/alot?utm_source=github.com&utm_medium=referral&utm_content=pazz/alot&utm_campaign=Badge_Coverage [codacy-coverage-img]: https://api.codacy.com/project/badge/Coverage/fa7c4a567cd546568a12e88c57f9dbd6 [codacy-grade]: https://www.codacy.com/app/patricktotzke/alot?utm_source=github.com&utm_medium=referral&utm_content=pazz/alot&utm_campaign=Badge_Grade diff --git a/alot/db/thread.py b/alot/db/thread.py index 907be861e..9b9b11c71 100644 --- a/alot/db/thread.py +++ b/alot/db/thread.py @@ -3,8 +3,10 @@ # For further details see the COPYING file from datetime import datetime +from ..helper import string_sanitize from .message import Message from ..settings.const import settings +from .utils import decode_header class Thread: @@ -44,11 +46,11 @@ def _refresh(self, thread): subject_type = settings.get('thread_subject') if subject_type == 'notmuch': - subject = thread.subject + subject = string_sanitize(thread.subject) elif subject_type == 'oldest': try: first_msg = list(thread.toplevel())[0] - subject = first_msg.header('subject') + subject = decode_header(first_msg.header('subject')) except (IndexError, LookupError): subject = '' self._subject = subject diff --git a/alot/helper.py b/alot/helper.py index 7d6c1dad3..122c16394 100644 --- a/alot/helper.py +++ b/alot/helper.py @@ -7,11 +7,11 @@ from datetime import datetime from collections import deque import logging -import mimetypes import os import re import shlex import subprocess +import unicodedata import email from email.mime.audio import MIMEAudio from email.mime.base import MIMEBase @@ -41,6 +41,17 @@ def split_commandstring(cmdstring): return shlex.split(cmdstring) +def unicode_printable(c): + """ + Checks if the given character is a printable Unicode character, i.e., not a + private/unassigned character and not a control character other than tab or + newline. + """ + if c in ('\n', '\t'): + return True + return unicodedata.category(c) not in ('Cc', 'Cn', 'Co') + + def string_sanitize(string, tab_width=8): r""" strips, and replaces non-printable characters @@ -57,7 +68,7 @@ def string_sanitize(string, tab_width=8): 'foo bar' """ - string = string.replace('\r', '') + string = ''.join([c for c in string if unicode_printable(c)]) lines = list() for line in string.split('\n'): @@ -397,34 +408,6 @@ def try_decode(blob): return blob.decode(guess_encoding(blob)) -def libmagic_version_at_least(version): - """ - checks if the libmagic library installed is more recent than a given - version. - - :param version: minimum version expected in the form XYY (i.e. 5.14 -> 514) - with XYY >= 513 - """ - if hasattr(magic, 'open'): - magic_wrapper = magic._libraries['magic'] - elif hasattr(magic, 'from_buffer'): - magic_wrapper = magic.libmagic - else: - raise Exception('Unknown magic API') - - if not hasattr(magic_wrapper, 'magic_version'): - # The magic_version function has been introduced in libmagic 5.13, - # if it's not present, we can't guess right, so let's assume False - return False - - # Depending on the libmagic/ctypes version, magic_version is a function or - # a callable: - if callable(magic_wrapper.magic_version): - return magic_wrapper.magic_version() >= version - - return magic_wrapper.magic_version >= version - - # TODO: make this work on blobs, not paths def mimewrap(path, filename=None, ctype=None): """Take the contents of the given path and wrap them into an email MIME @@ -443,18 +426,7 @@ def mimewrap(path, filename=None, ctype=None): with open(path, 'rb') as f: content = f.read() - if not ctype: - ctype = guess_mimetype(content) - # libmagic < 5.12 incorrectly detects excel/powerpoint files as - # 'application/msword' (see #179 and #186 in libmagic bugtracker) - # This is a workaround, based on file extension, useful as long - # as distributions still ship libmagic 5.11. - if (ctype == 'application/msword' and - not libmagic_version_at_least(513)): - mimetype, _ = mimetypes.guess_type(path) - if mimetype: - ctype = mimetype - + ctype = ctype or guess_mimetype(content) maintype, subtype = ctype.split('/', 1) if maintype == 'text': part = MIMEText(content.decode(guess_encoding(content), 'replace'), diff --git a/poetry.lock b/poetry.lock index d50db9b14..9ed65d54c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,5 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + [[package]] name = "alabaster" @@ -73,13 +74,13 @@ pytz = ">=2015.7" [[package]] name = "certifi" -version = "2023.7.22" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] @@ -245,13 +246,13 @@ idna = ">=2.5" [[package]] name = "idna" -version = "3.4" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] [[package]] @@ -593,18 +594,18 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<5)"] [[package]] name = "setuptools" -version = "59.6.0" +version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "setuptools-59.6.0-py3-none-any.whl", hash = "sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e"}, - {file = "setuptools-59.6.0.tar.gz", hash = "sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=8.2)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-inline-tabs", "sphinxcontrib-towncrier"] -testing = ["flake8-2020", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "paver", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy", "pytest-virtualenv (>=1.2.7)", "pytest-xdist", "sphinx", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -837,17 +838,17 @@ files = [ [[package]] name = "urllib3" -version = "1.26.16" +version = "1.26.19" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-1.26.19-py2.py3-none-any.whl", hash = "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3"}, + {file = "urllib3-1.26.19.tar.gz", hash = "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] @@ -880,18 +881,18 @@ docs = ["mock"] [[package]] name = "zipp" -version = "3.6.0" +version = "3.19.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, - {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, + {file = "zipp-3.19.1-py3-none-any.whl", hash = "sha256:2828e64edb5386ea6a52e7ba7cdb17bb30a73a858f5eb6eb93d8d36f5ea26091"}, + {file = "zipp-3.19.1.tar.gz", hash = "sha256:35427f6d5594f4acf82d25541438348c26736fa9b3afa2754bcd63cdb99d8e8f"}, ] [package.extras] -docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] -testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [[package]] name = "zope-interface" diff --git a/tests/test_helper.py b/tests/test_helper.py index 5bc801c92..1dd5666a4 100644 --- a/tests/test_helper.py +++ b/tests/test_helper.py @@ -167,6 +167,12 @@ def test_tabs(self): actual = helper.string_sanitize(base) self.assertEqual(actual, expected) + def test_control_characters(self): + base = 'foo\u009dbar\u0007\rtest' + expected = 'foobartest' + actual = helper.string_sanitize(base) + self.assertEqual(actual, expected) + class TestStringDecode(unittest.TestCase):