Merge branch 'master' into dependabot/pip/twisted-23.10.0

README.md

@@ -1,4 +1,4 @@
-[![Build Status][travis-img]][travis]
+[![Build Status]][ghactions]
 [![Code Climate][codeclimate-img]][codeclimate]
 [![Codacy Grade][codacy-grade-img]][codacy-grade]
 [![Codacy Coverage][codacy-coverage-img]][codacy-coverage]
@@ -59,8 +59,8 @@ See the [manual][docs] for more usage info.
 [FAQ]: http://alot.readthedocs.io/en/latest/faq.html
 [features]: https://github.com/pazz/alot/issues?labels=feature
 
-[travis]: https://travis-ci.com/pazz/alot
-[travis-img]: https://travis-ci.com/pazz/alot.svg?branch=master
+[Build Status]: https://github.com/pazz/alot/actions/workflows/check.yml/badge.svg
+[ghactions]: https://github.com/pazz/alot/actions
 [codacy-coverage]: https://www.codacy.com/app/patricktotzke/alot?utm_source=github.com&utm_medium=referral&utm_content=pazz/alot&utm_campaign=Badge_Coverage
 [codacy-coverage-img]: https://api.codacy.com/project/badge/Coverage/fa7c4a567cd546568a12e88c57f9dbd6
 [codacy-grade]: https://www.codacy.com/app/patricktotzke/alot?utm_source=github.com&utm_medium=referral&utm_content=pazz/alot&utm_campaign=Badge_Grade

alot/db/thread.py

@@ -3,8 +3,10 @@
 # For further details see the COPYING file
 from datetime import datetime
 
+from ..helper import string_sanitize
 from .message import Message
 from ..settings.const import settings
+from .utils import decode_header
 
 
 class Thread:
@@ -44,11 +46,11 @@ def _refresh(self, thread):
 
         subject_type = settings.get('thread_subject')
         if subject_type == 'notmuch':
-            subject = thread.subject
+            subject = string_sanitize(thread.subject)
         elif subject_type == 'oldest':
             try:
                 first_msg = list(thread.toplevel())[0]
-                subject = first_msg.header('subject')
+                subject = decode_header(first_msg.header('subject'))
             except (IndexError, LookupError):
                 subject = ''
         self._subject = subject

alot/helper.py

@@ -7,11 +7,11 @@
 from datetime import datetime
 from collections import deque
 import logging
-import mimetypes
 import os
 import re
 import shlex
 import subprocess
+import unicodedata
 import email
 from email.mime.audio import MIMEAudio
 from email.mime.base import MIMEBase
@@ -41,6 +41,17 @@ def split_commandstring(cmdstring):
     return shlex.split(cmdstring)
 
 
+def unicode_printable(c):
+    """
+    Checks if the given character is a printable Unicode character, i.e., not a
+    private/unassigned character and not a control character other than tab or
+    newline.
+    """
+    if c in ('\n', '\t'):
+        return True
+    return unicodedata.category(c) not in ('Cc', 'Cn', 'Co')
+
+
 def string_sanitize(string, tab_width=8):
     r"""
     strips, and replaces non-printable characters
@@ -57,7 +68,7 @@ def string_sanitize(string, tab_width=8):
     'foo             bar'
     """
 
-    string = string.replace('\r', '')
+    string = ''.join([c for c in string if unicode_printable(c)])
 
     lines = list()
     for line in string.split('\n'):
@@ -397,34 +408,6 @@ def try_decode(blob):
     return blob.decode(guess_encoding(blob))
 
 
-def libmagic_version_at_least(version):
-    """
-    checks if the libmagic library installed is more recent than a given
-    version.
-
-    :param version: minimum version expected in the form XYY (i.e. 5.14 -> 514)
-                    with XYY >= 513
-    """
-    if hasattr(magic, 'open'):
-        magic_wrapper = magic._libraries['magic']
-    elif hasattr(magic, 'from_buffer'):
-        magic_wrapper = magic.libmagic
-    else:
-        raise Exception('Unknown magic API')
-
-    if not hasattr(magic_wrapper, 'magic_version'):
-        # The magic_version function has been introduced in libmagic 5.13,
-        # if it's not present, we can't guess right, so let's assume False
-        return False
-
-    # Depending on the libmagic/ctypes version, magic_version is a function or
-    # a callable:
-    if callable(magic_wrapper.magic_version):
-        return magic_wrapper.magic_version() >= version
-
-    return magic_wrapper.magic_version >= version
-
-
 # TODO: make this work on blobs, not paths
 def mimewrap(path, filename=None, ctype=None):
     """Take the contents of the given path and wrap them into an email MIME
@@ -443,18 +426,7 @@ def mimewrap(path, filename=None, ctype=None):
 
     with open(path, 'rb') as f:
         content = f.read()
-    if not ctype:
-        ctype = guess_mimetype(content)
-        # libmagic < 5.12 incorrectly detects excel/powerpoint files as
-        # 'application/msword' (see #179 and #186 in libmagic bugtracker)
-        # This is a workaround, based on file extension, useful as long
-        # as distributions still ship libmagic 5.11.
-        if (ctype == 'application/msword' and
-                not libmagic_version_at_least(513)):
-            mimetype, _ = mimetypes.guess_type(path)
-            if mimetype:
-                ctype = mimetype
-
+    ctype = ctype or guess_mimetype(content)
     maintype, subtype = ctype.split('/', 1)
     if maintype == 'text':
         part = MIMEText(content.decode(guess_encoding(content), 'replace'),

tests/test_helper.py

@@ -167,6 +167,12 @@ def test_tabs(self):
         actual = helper.string_sanitize(base)
         self.assertEqual(actual, expected)
 
+    def test_control_characters(self):
+        base = 'foo\u009dbar\u0007\rtest'
+        expected = 'foobartest'
+        actual = helper.string_sanitize(base)
+        self.assertEqual(actual, expected)
+
 
 class TestStringDecode(unittest.TestCase):