[PATCH 0/3] Refactored code roundtrip via signatures

[PATCH 0/3] Refactored code roundtrip via signatures

[Konstantin Ryabitsev]

Another roundtrip via signatures list, using openpgp.

Konstantin Ryabitsev (2):
  Add -f to "b4 mbox" to filter dupes
  Save to/cc headers as-is for tracking

Kyle Meyer (1):
  Loosen compatible release identifiers for install_requires

 b4/__init__.py | 11 ++++++++---
 b4/command.py  |  2 ++
 b4/mbox.py     | 15 +++++++++++----
 b4/pr.py       |  4 ++--
 man/b4.5       |  5 ++++-
 man/b4.5.rst   |  1 +
 setup.py       |  6 +++---
 7 files changed, 31 insertions(+), 13 deletions(-)

-- 
2.30.2

[PATCH 1/3] Add -f to "b4 mbox" to filter dupes

[Konstantin Ryabitsev]

When saving to a maildir, add option to filter out dupes. Note, that
this requires going through the entire maildir to collect message-ids,
so it's not going to be a great experience on large maildirs.

Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
---
 b4/command.py |  2 ++
 b4/mbox.py    | 11 +++++++++--
 man/b4.5      |  5 ++++-
 man/b4.5.rst  |  1 +
 4 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/b4/command.py b/b4/command.py
index 0c87df9..59fefbf 100644
--- a/b4/command.py
+++ b/b4/command.py
@@ -84,6 +84,8 @@ def cmd():
     # b4 mbox
     sp_mbox = subparsers.add_parser('mbox', help='Download a thread as an mbox file')
     cmd_mbox_common_opts(sp_mbox)
+    sp_mbox.add_argument('-f', '--filter-dupes', dest='filterdupes', action='store_true', default=False,
+                         help='When adding messages to existing maildir, filter out duplicates')
     sp_mbox.set_defaults(func=cmd_mbox)
 
     # b4 am
diff --git a/b4/mbox.py b/b4/mbox.py
index 226303e..d84d390 100644
--- a/b4/mbox.py
+++ b/b4/mbox.py
@@ -549,9 +549,16 @@ def main(cmdargs):
             and os.path.isdir(os.path.join(cmdargs.outdir, 'cur'))
             and os.path.isdir(os.path.join(cmdargs.outdir, 'tmp'))):
         mdr = mailbox.Maildir(cmdargs.outdir)
+        have_msgids = set()
+        added = 0
+        if cmdargs.filterdupes:
+            for emsg in mdr:
+                have_msgids.add(b4.LoreMessage.get_clean_msgid(emsg))
         for msg in mbx:
-            mdr.add(msg)
-        logger.info('Added to maildir %s', cmdargs.outdir)
+            if b4.LoreMessage.get_clean_msgid(msg) not in have_msgids:
+                added += 1
+                mdr.add(msg)
+        logger.info('Added to %s messages to maildir %s', added, cmdargs.outdir)
         mbx.close()
         os.unlink(threadfile)
         return
diff --git a/man/b4.5 b/man/b4.5
index 350c363..496525d 100644
--- a/man/b4.5
+++ b/man/b4.5
@@ -102,6 +102,9 @@ Instead of grabbing a thread from lore, process this mbox file
 .TP
 .B \-C\fP,\fB  \-\-no\-cache
 Do not use local cache
+.TP
+.B \-f\fP,\fB  \-\-filter\-dupes
+When adding messages to existing maildir, filter out duplicates
 .UNINDENT
 .UNINDENT
 .sp
@@ -341,7 +344,7 @@ Default configuration, with explanations:
 .ft C
 [b4]
    # Where to look up threads by message id
-   midmask = https://lore.kernel.org/r/%s\(aq
+   midmask = https://lore.kernel.org/r/%s
    #
    # When recording Link: trailers, use this mask
    linkmask = https://lore.kernel.org/r/%s
diff --git a/man/b4.5.rst b/man/b4.5.rst
index e21ed71..ee05675 100644
--- a/man/b4.5.rst
+++ b/man/b4.5.rst
@@ -63,6 +63,7 @@ optional arguments:
   -m LOCALMBOX, --use-local-mbox LOCALMBOX
                         Instead of grabbing a thread from lore, process this mbox file
   -C, --no-cache        Do not use local cache
+  -f, --filter-dupes    When adding messages to existing maildir, filter out duplicates
 
 *Example*: b4 mbox 20200313231252.64999-1-keescook@chromium.org
 
-- 
2.30.2

[PATCH 2/3] Save to/cc headers as-is for tracking

[Konstantin Ryabitsev]

If we clean the to/cc headers to get rid of all unicode escaping, we run
into a Python bug that is unable to properly parse addresses, e.g.:

In [5]: from email import utils

In [6]: utils.getaddresses(['foo <foo@bar.com>'])
Out[6]: [('foo', 'foo@bar.com')]

In [7]: utils.getaddresses(['Shuming [范書銘] <shumingf@realtek.com>'])
Out[7]:
[('', 'Shuming'),
 ('', ''),
 ('', '范書銘'),
 ('', ''),
 ('', 'shumingf@realtek.com')]

If we store the headers as-is from the original message, we are less
likely to run into this bug, as all non-ascii sequences should be
qp-escaped in the original headers:

=?big5?B?U2h1bWluZyBbrVOu0bvKXQ==?= <shumingf@realtek.com>

This doesn't fix the underlying bug in Python, but works around it.

Reported-by: Mark Brown <broonie@kernel.org>
Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
---
 b4/__init__.py | 11 ++++++++---
 b4/mbox.py     |  4 ++--
 b4/pr.py       |  4 ++--
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/b4/__init__.py b/b4/__init__.py
index ee07f16..32b5c02 100644
--- a/b4/__init__.py
+++ b/b4/__init__.py
@@ -2375,11 +2375,16 @@ def git_get_toplevel(path=None):
     return topdir
 
 
-def format_addrs(pairs):
+def format_addrs(pairs, clean=True):
     addrs = set()
     for pair in pairs:
-        # Remove any quoted-printable header junk from the name
-        addrs.add(email.utils.formataddr((LoreMessage.clean_header(pair[0]), LoreMessage.clean_header(pair[1]))))
+        pair = list(pair)
+        if pair[0] == pair[1]:
+            pair[0] = ''
+        if clean:
+            # Remove any quoted-printable header junk from the name
+            pair[0] = LoreMessage.clean_header(pair[0])
+        addrs.add(email.utils.formataddr(pair))  # noqa
     return ', '.join(addrs)
 
 
diff --git a/b4/mbox.py b/b4/mbox.py
index d84d390..d3bde25 100644
--- a/b4/mbox.py
+++ b/b4/mbox.py
@@ -294,8 +294,8 @@ def thanks_record_am(lser, cherrypick=None):
         'subject': lmsg.full_subject,
         'fromname': lmsg.fromname,
         'fromemail': lmsg.fromemail,
-        'to': b4.format_addrs(allto),
-        'cc': b4.format_addrs(allcc),
+        'to': b4.format_addrs(allto, clean=False),
+        'cc': b4.format_addrs(allcc, clean=False),
         'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
         'sentdate': b4.LoreMessage.clean_header(lmsg.msg['Date']),
         'quote': b4.make_quote(lmsg.body, maxlines=5),
diff --git a/b4/pr.py b/b4/pr.py
index 0ff68f8..5e6c7a1 100644
--- a/b4/pr.py
+++ b/b4/pr.py
@@ -225,8 +225,8 @@ def thanks_record_pr(lmsg):
         'subject': lmsg.full_subject,
         'fromname': lmsg.fromname,
         'fromemail': lmsg.fromemail,
-        'to': b4.format_addrs(allto),
-        'cc': b4.format_addrs(allcc),
+        'to': b4.format_addrs(allto, clean=False),
+        'cc': b4.format_addrs(allcc, clean=False),
         'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
         'remote': lmsg.pr_repo,
         'ref': lmsg.pr_ref,
-- 
2.30.2

[PATCH 3/3] Loosen compatible release identifiers for install_requires

[Konstantin Ryabitsev]

From: Kyle Meyer <kyle@kyleam.com>

The install_requires entries use a compatible release operator.  As an
example, "requests~=2.24.0" maps to a requirement of ">= 2.24.0 and ==
2.24.*".  With the current version of requests (2.25.1), this leads to
a ContextualVersionConflict failure at runtime.

Allowing only Z to tick in version X.Y.Z seems unnecessarily strict
unless there are known problems with a particular release, and it
makes it more difficult for distributions to package b4.  Drop the
trailing digit from all of the version identifiers, allowing both Y
and Z to increase.

Signed-off-by: Kyle Meyer <kyle@kyleam.com>
Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 358e6a7..a21ec76 100644
--- a/setup.py
+++ b/setup.py
@@ -41,9 +41,9 @@ setup(
     data_files = [('share/man/man5', ['man/b4.5'])],
     keywords=['git', 'lore.kernel.org', 'patches'],
     install_requires=[
-        'requests~=2.24.0',
-        'dkimpy~=1.0.5',
-        'dnspython~=2.0.0',
+        'requests~=2.24',
+        'dkimpy~=1.0',
+        'dnspython~=2.0',
     ],
     python_requires='>=3.6',
     entry_points={
-- 
2.30.2

[PATCH 0/3] One more test with openpgp-sha256

[Konstantin Ryabitsev]

Another test with roundtrip via signatures list.

Konstantin Ryabitsev (2):
  Add -f to "b4 mbox" to filter dupes
  Save to/cc headers as-is for tracking

Kyle Meyer (1):
  Loosen compatible release identifiers for install_requires

 b4/__init__.py | 11 ++++++++---
 b4/command.py  |  2 ++
 b4/mbox.py     | 15 +++++++++++----
 b4/pr.py       |  4 ++--
 man/b4.5       |  5 ++++-
 man/b4.5.rst   |  1 +
 setup.py       |  6 +++---
 7 files changed, 31 insertions(+), 13 deletions(-)

-- 
2.30.2

[PATCH 2/3] Save to/cc headers as-is for tracking

[Konstantin Ryabitsev]

If we clean the to/cc headers to get rid of all unicode escaping, we run
into a Python bug that is unable to properly parse addresses, e.g.:

In [5]: from email import utils

In [6]: utils.getaddresses(['foo <foo@bar.com>'])
Out[6]: [('foo', 'foo@bar.com')]

In [7]: utils.getaddresses(['Shuming [范書銘] <shumingf@realtek.com>'])
Out[7]:
[('', 'Shuming'),
 ('', ''),
 ('', '范書銘'),
 ('', ''),
 ('', 'shumingf@realtek.com')]

If we store the headers as-is from the original message, we are less
likely to run into this bug, as all non-ascii sequences should be
qp-escaped in the original headers:

=?big5?B?U2h1bWluZyBbrVOu0bvKXQ==?= <shumingf@realtek.com>

This doesn't fix the underlying bug in Python, but works around it.

Reported-by: Mark Brown <broonie@kernel.org>
Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
---
 b4/__init__.py | 11 ++++++++---
 b4/mbox.py     |  4 ++--
 b4/pr.py       |  4 ++--
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/b4/__init__.py b/b4/__init__.py
index ee07f16..32b5c02 100644
--- a/b4/__init__.py
+++ b/b4/__init__.py
@@ -2375,11 +2375,16 @@ def git_get_toplevel(path=None):
     return topdir
 
 
-def format_addrs(pairs):
+def format_addrs(pairs, clean=True):
     addrs = set()
     for pair in pairs:
-        # Remove any quoted-printable header junk from the name
-        addrs.add(email.utils.formataddr((LoreMessage.clean_header(pair[0]), LoreMessage.clean_header(pair[1]))))
+        pair = list(pair)
+        if pair[0] == pair[1]:
+            pair[0] = ''
+        if clean:
+            # Remove any quoted-printable header junk from the name
+            pair[0] = LoreMessage.clean_header(pair[0])
+        addrs.add(email.utils.formataddr(pair))  # noqa
     return ', '.join(addrs)
 
 
diff --git a/b4/mbox.py b/b4/mbox.py
index d84d390..d3bde25 100644
--- a/b4/mbox.py
+++ b/b4/mbox.py
@@ -294,8 +294,8 @@ def thanks_record_am(lser, cherrypick=None):
         'subject': lmsg.full_subject,
         'fromname': lmsg.fromname,
         'fromemail': lmsg.fromemail,
-        'to': b4.format_addrs(allto),
-        'cc': b4.format_addrs(allcc),
+        'to': b4.format_addrs(allto, clean=False),
+        'cc': b4.format_addrs(allcc, clean=False),
         'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
         'sentdate': b4.LoreMessage.clean_header(lmsg.msg['Date']),
         'quote': b4.make_quote(lmsg.body, maxlines=5),
diff --git a/b4/pr.py b/b4/pr.py
index 0ff68f8..5e6c7a1 100644
--- a/b4/pr.py
+++ b/b4/pr.py
@@ -225,8 +225,8 @@ def thanks_record_pr(lmsg):
         'subject': lmsg.full_subject,
         'fromname': lmsg.fromname,
         'fromemail': lmsg.fromemail,
-        'to': b4.format_addrs(allto),
-        'cc': b4.format_addrs(allcc),
+        'to': b4.format_addrs(allto, clean=False),
+        'cc': b4.format_addrs(allcc, clean=False),
         'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
         'remote': lmsg.pr_repo,
         'ref': lmsg.pr_ref,
-- 
2.30.2

[PATCH 0/3] Another test, with ed25519-sha256

[Konstantin Ryabitsev]

This series is signed with an ed25519 key.

Konstantin Ryabitsev (2):
  Add -f to "b4 mbox" to filter dupes
  Save to/cc headers as-is for tracking

Kyle Meyer (1):
  Loosen compatible release identifiers for install_requires

 b4/__init__.py | 11 ++++++++---
 b4/command.py  |  2 ++
 b4/mbox.py     | 15 +++++++++++----
 b4/pr.py       |  4 ++--
 man/b4.5       |  5 ++++-
 man/b4.5.rst   |  1 +
 setup.py       |  6 +++---
 7 files changed, 31 insertions(+), 13 deletions(-)

-- 
2.31.1

[PATCH 2/3] Save to/cc headers as-is for tracking

[Konstantin Ryabitsev]

If we clean the to/cc headers to get rid of all unicode escaping, we run
into a Python bug that is unable to properly parse addresses, e.g.:

In [5]: from email import utils

In [6]: utils.getaddresses(['foo <foo@bar.com>'])
Out[6]: [('foo', 'foo@bar.com')]

In [7]: utils.getaddresses(['Shuming [范書銘] <shumingf@realtek.com>'])
Out[7]:
[('', 'Shuming'),
 ('', ''),
 ('', '范書銘'),
 ('', ''),
 ('', 'shumingf@realtek.com')]

If we store the headers as-is from the original message, we are less
likely to run into this bug, as all non-ascii sequences should be
qp-escaped in the original headers:

=?big5?B?U2h1bWluZyBbrVOu0bvKXQ==?= <shumingf@realtek.com>

This doesn't fix the underlying bug in Python, but works around it.

Reported-by: Mark Brown <broonie@kernel.org>
Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
---
 b4/__init__.py | 11 ++++++++---
 b4/mbox.py     |  4 ++--
 b4/pr.py       |  4 ++--
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/b4/__init__.py b/b4/__init__.py
index ee07f16..32b5c02 100644
--- a/b4/__init__.py
+++ b/b4/__init__.py
@@ -2375,11 +2375,16 @@ def git_get_toplevel(path=None):
     return topdir
 
 
-def format_addrs(pairs):
+def format_addrs(pairs, clean=True):
     addrs = set()
     for pair in pairs:
-        # Remove any quoted-printable header junk from the name
-        addrs.add(email.utils.formataddr((LoreMessage.clean_header(pair[0]), LoreMessage.clean_header(pair[1]))))
+        pair = list(pair)
+        if pair[0] == pair[1]:
+            pair[0] = ''
+        if clean:
+            # Remove any quoted-printable header junk from the name
+            pair[0] = LoreMessage.clean_header(pair[0])
+        addrs.add(email.utils.formataddr(pair))  # noqa
     return ', '.join(addrs)
 
 
diff --git a/b4/mbox.py b/b4/mbox.py
index d84d390..d3bde25 100644
--- a/b4/mbox.py
+++ b/b4/mbox.py
@@ -294,8 +294,8 @@ def thanks_record_am(lser, cherrypick=None):
         'subject': lmsg.full_subject,
         'fromname': lmsg.fromname,
         'fromemail': lmsg.fromemail,
-        'to': b4.format_addrs(allto),
-        'cc': b4.format_addrs(allcc),
+        'to': b4.format_addrs(allto, clean=False),
+        'cc': b4.format_addrs(allcc, clean=False),
         'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
         'sentdate': b4.LoreMessage.clean_header(lmsg.msg['Date']),
         'quote': b4.make_quote(lmsg.body, maxlines=5),
diff --git a/b4/pr.py b/b4/pr.py
index 0ff68f8..5e6c7a1 100644
--- a/b4/pr.py
+++ b/b4/pr.py
@@ -225,8 +225,8 @@ def thanks_record_pr(lmsg):
         'subject': lmsg.full_subject,
         'fromname': lmsg.fromname,
         'fromemail': lmsg.fromemail,
-        'to': b4.format_addrs(allto),
-        'cc': b4.format_addrs(allcc),
+        'to': b4.format_addrs(allto, clean=False),
+        'cc': b4.format_addrs(allcc, clean=False),
         'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
         'remote': lmsg.pr_repo,
         'ref': lmsg.pr_ref,
-- 
2.31.1

[PATCH 0/3] Test with new signature scheme

[Konstantin Ryabitsev]

This is just a test, please ignore.

Konstantin Ryabitsev (2):
  Add -f to "b4 mbox" to filter dupes
  Save to/cc headers as-is for tracking

Kyle Meyer (1):
  Loosen compatible release identifiers for install_requires

 b4/__init__.py | 11 ++++++++---
 b4/command.py  |  2 ++
 b4/mbox.py     | 15 +++++++++++----
 b4/pr.py       |  4 ++--
 man/b4.5       |  5 ++++-
 man/b4.5.rst   |  1 +
 setup.py       |  6 +++---
 7 files changed, 31 insertions(+), 13 deletions(-)

-- 
2.31.1

[PATCH 2/3] Save to/cc headers as-is for tracking

[Konstantin Ryabitsev]

If we clean the to/cc headers to get rid of all unicode escaping, we run
into a Python bug that is unable to properly parse addresses, e.g.:

In [5]: from email import utils

In [6]: utils.getaddresses(['foo <foo@bar.com>'])
Out[6]: [('foo', 'foo@bar.com')]

In [7]: utils.getaddresses(['Shuming [范書銘] <shumingf@realtek.com>'])
Out[7]:
[('', 'Shuming'),
 ('', ''),
 ('', '范書銘'),
 ('', ''),
 ('', 'shumingf@realtek.com')]

If we store the headers as-is from the original message, we are less
likely to run into this bug, as all non-ascii sequences should be
qp-escaped in the original headers:

=?big5?B?U2h1bWluZyBbrVOu0bvKXQ==?= <shumingf@realtek.com>

This doesn't fix the underlying bug in Python, but works around it.

Reported-by: Mark Brown <broonie@kernel.org>
Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
---
 b4/__init__.py | 11 ++++++++---
 b4/mbox.py     |  4 ++--
 b4/pr.py       |  4 ++--
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/b4/__init__.py b/b4/__init__.py
index ee07f16..32b5c02 100644
--- a/b4/__init__.py
+++ b/b4/__init__.py
@@ -2375,11 +2375,16 @@ def git_get_toplevel(path=None):
     return topdir
 
 
-def format_addrs(pairs):
+def format_addrs(pairs, clean=True):
     addrs = set()
     for pair in pairs:
-        # Remove any quoted-printable header junk from the name
-        addrs.add(email.utils.formataddr((LoreMessage.clean_header(pair[0]), LoreMessage.clean_header(pair[1]))))
+        pair = list(pair)
+        if pair[0] == pair[1]:
+            pair[0] = ''
+        if clean:
+            # Remove any quoted-printable header junk from the name
+            pair[0] = LoreMessage.clean_header(pair[0])
+        addrs.add(email.utils.formataddr(pair))  # noqa
     return ', '.join(addrs)
 
 
diff --git a/b4/mbox.py b/b4/mbox.py
index d84d390..d3bde25 100644
--- a/b4/mbox.py
+++ b/b4/mbox.py
@@ -294,8 +294,8 @@ def thanks_record_am(lser, cherrypick=None):
         'subject': lmsg.full_subject,
         'fromname': lmsg.fromname,
         'fromemail': lmsg.fromemail,
-        'to': b4.format_addrs(allto),
-        'cc': b4.format_addrs(allcc),
+        'to': b4.format_addrs(allto, clean=False),
+        'cc': b4.format_addrs(allcc, clean=False),
         'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
         'sentdate': b4.LoreMessage.clean_header(lmsg.msg['Date']),
         'quote': b4.make_quote(lmsg.body, maxlines=5),
diff --git a/b4/pr.py b/b4/pr.py
index 0ff68f8..5e6c7a1 100644
--- a/b4/pr.py
+++ b/b4/pr.py
@@ -225,8 +225,8 @@ def thanks_record_pr(lmsg):
         'subject': lmsg.full_subject,
         'fromname': lmsg.fromname,
         'fromemail': lmsg.fromemail,
-        'to': b4.format_addrs(allto),
-        'cc': b4.format_addrs(allcc),
+        'to': b4.format_addrs(allto, clean=False),
+        'cc': b4.format_addrs(allcc, clean=False),
         'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
         'remote': lmsg.pr_repo,
         'ref': lmsg.pr_ref,
-- 
2.31.1