fix(common): set consistent timestamps on template-generated files

common/tools/cointool.py

@@ -3,14 +3,14 @@
 
 import datetime
 import fnmatch
-import glob
 import json
 import logging
 import os
 import re
 import sys
 from collections import defaultdict
 from hashlib import sha256
+from pathlib import Path
 from typing import Any, Callable, Iterator, TextIO, cast
 
 import click
@@ -133,13 +133,13 @@ def supported_on(device: str, coins: Coins) -> Iterator[Coin]:
 
 
 def render_file(
-    src: str, dst: TextIO, coins: CoinsInfo, support_info: SupportInfo
+    src: Path, dst: Path, coins: CoinsInfo, support_info: SupportInfo
 ) -> None:
     """Renders `src` template into `dst`.
 
     `src` is a filename, `dst` is an open file object.
     """
-    template = mako.template.Template(filename=src)
+    template = mako.template.Template(filename=str(src.resolve()))
     eth_defs_date = datetime.datetime.fromisoformat(
         DEFINITIONS_TIMESTAMP_PATH.read_text().strip()
     )
@@ -150,7 +150,9 @@ def render_file(
         **coins,
         **MAKO_FILTERS,
     )
-    dst.write(result)
+    dst.write_text(str(result))
+    src_stat = src.stat()
+    os.utime(dst, ns=(src_stat.st_atime_ns, src_stat.st_mtime_ns))
 
 
 # ====== validation functions ======
@@ -840,13 +842,13 @@ def modify_coin(coin: Coin) -> Coin:
 
 @cli.command()
 # fmt: off
-@click.argument("paths", metavar="[path]...", nargs=-1)
-@click.option("-o", "--outfile", type=click.File("w"), help="Alternate output file")
+@click.argument("paths", type=click.Path(path_type=Path), metavar="[path]...", nargs=-1)
+@click.option("-o", "--outfile", type=click.Path(dir_okay=False, writable=True, path_type=Path), help="Alternate output file")
 @click.option("-v", "--verbose", is_flag=True, help="Print rendered file names")
 @click.option("-b", "--bitcoin-only", is_flag=True, help="Accept only Bitcoin coins")
 # fmt: on
 def render(
-    paths: tuple[str, ...], outfile: TextIO, verbose: bool, bitcoin_only: bool
+    paths: tuple[Path, ...], outfile: Path, verbose: bool, bitcoin_only: bool
 ) -> None:
     """Generate source code from Mako templates.
 
@@ -882,7 +884,7 @@ def render(
     for key, value in support_info.items():
         support_info[key] = Munch(value)
 
-    def do_render(src: str, dst: TextIO) -> None:
+    def do_render(src: Path, dst: Path) -> None:
         if verbose:
             click.echo(f"Rendering {src} => {dst.name}")
         render_file(src, dst, defs, support_info)
@@ -894,25 +896,23 @@ def do_render(src: str, dst: TextIO) -> None:
 
     # find files in directories
     if not paths:
-        paths = (".",)
+        paths = (Path(),)
 
-    files: list[str] = []
+    files: list[Path] = []
     for path in paths:
-        if not os.path.exists(path):
+        if not path.exists():
             click.echo(f"Path {path} does not exist")
-        elif os.path.isdir(path):
-            files += glob.glob(os.path.join(path, "*.mako"))
+        elif path.is_dir():
+            files.extend(path.glob("*.mako"))
         else:
             files.append(path)
 
     # render each file
     for file in files:
-        if not file.endswith(".mako"):
+        if not file.suffix == ".mako":
             click.echo(f"File {file} does not end with .mako")
         else:
-            target = file[: -len(".mako")]
-            with open(target, "w") as dst:
-                do_render(file, dst)
+            do_render(file, file.parent / file.stem)
 
 
 @cli.command()

core/Makefile

@@ -1,6 +1,12 @@
 .PHONY: vendor
 
-JOBS = 4
+# get the number of CPU cores in a "portable" manner
+# (accounting for darwin and big.LITTLE archs)
+JOBS = $(shell sysctl -n hw.perflevel0.physicalcpu 2>/dev/null ||\
+               sysctl -n hw.physicalcpu            2>/dev/null ||\
+               sysctl -n hw.ncpu                   2>/dev/null ||\
+               nproc                               2>/dev/null ||\
+               echo 4)
 MAKE = make -j $(JOBS)
 SCONS = scons -Q -j $(JOBS)
 

core/SConscript.firmware

@@ -489,8 +489,10 @@ qstr_protobuf = env.Command(
 
 qstr_micropython = 'vendor/micropython/py/qstrdefs.h'
 
+micropy_defines = env.MicroPyDefines(source=SOURCE_QSTR)
+
 qstr_collected = env.CollectQstr(
-    target='genhdr/qstrdefs.collected.h', source=SOURCE_QSTR)
+    target='genhdr/qstrdefs.collected.h', source=micropy_defines)
 
 qstr_preprocessed = env.PreprocessQstr(
     target='genhdr/qstrdefs.preprocessed.h',
@@ -506,16 +508,16 @@ env.Ignore(qstr_collected, qstr_generated)
 #
 
 moduledefs_collected = env.CollectModules(
-    target='genhdr/moduledefs.collected.h', source=SOURCE_QSTR)
+    target='genhdr/moduledefs.collected.h', source=micropy_defines)
 
 hdr_moduledefs = env.Command(
     target='genhdr/moduledefs.h',
     source=moduledefs_collected,
     action='$MAKEMODULEDEFS $SOURCE > $TARGET', )
 
-env.Ignore(moduledefs_collected, moduledefs_collected)
-env.Ignore(moduledefs_collected, qstr_generated)
-env.Ignore(moduledefs_collected, hdr_moduledefs)
+env.Ignore(micropy_defines, micropy_defines)
+env.Ignore(micropy_defines, qstr_generated)
+env.Ignore(micropy_defines, hdr_moduledefs)
 
 #
 # Micropython version

core/SConscript.unix

@@ -564,8 +564,10 @@ qstr_protobuf = env.Command(
 
 qstr_micropython = 'vendor/micropython/py/qstrdefs.h'
 
+micropy_defines = env.MicroPyDefines(source=SOURCE_QSTR)
+
 qstr_collected = env.CollectQstr(
-    target='genhdr/qstrdefs.collected.h', source=SOURCE_QSTR)
+    target='genhdr/qstrdefs.collected.h', source=micropy_defines)
 
 qstr_preprocessed = env.PreprocessQstr(
     target='genhdr/qstrdefs.preprocessed.h',
@@ -581,7 +583,7 @@ env.Ignore(qstr_collected, qstr_generated)
 #
 
 moduledefs_collected = env.CollectModules(
-    target='genhdr/moduledefs.collected.h', source=SOURCE_QSTR)
+    target='genhdr/moduledefs.collected.h', source=micropy_defines)
 
 
 hdr_moduledefs = env.Command(
@@ -602,10 +604,10 @@ hdr_version = env.Command(
     source='',
     action='$MAKEVERSIONHDR $TARGET', )
 
-env.Ignore(hdr_moduledefs, hdr_moduledefs)
-env.Ignore(hdr_moduledefs, qstr_collected)
-env.Ignore(hdr_moduledefs, qstr_preprocessed)
-env.Ignore(hdr_moduledefs, qstr_generated)
+env.Ignore(micropy_defines, hdr_moduledefs)
+env.Ignore(micropy_defines, qstr_collected)
+env.Ignore(micropy_defines, qstr_preprocessed)
+env.Ignore(micropy_defines, qstr_generated)
 
 #
 # Frozen modules

core/SConstruct

@@ -1,4 +1,7 @@
 # pylint: disable=E0602
+Decider('content-timestamp')
+SetOption('max_drift', 10)
+SetOption('implicit_cache', 1)
 
 SConscript('SConscript.boardloader', variant_dir='build/boardloader', duplicate=False)
 SConscript('SConscript.bootloader', variant_dir='build/bootloader', duplicate=False)

core/site_scons/site_tools/micropython/__init__.py

@@ -8,9 +8,14 @@ def generate(env):
         MODULECOL="site_scons/site_tools/micropython/moduledefs.py",
     )
 
+    env["BUILDERS"]["MicroPyDefines"] = SCons.Builder.Builder(
+        action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCE > $TARGET",
+        suffix=".upydef",
+        single_source=True,
+    )
+
     env["BUILDERS"]["CollectQstr"] = SCons.Builder.Builder(
-        action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES"
-        " | $PYTHON $QSTRCOL > $TARGET"
+        action="cat $SOURCES | perl -nle 'print \"Q($1)\" while /MP_QSTR_(\\w+)/g' > $TARGET"
     )
 
     env["BUILDERS"]["PreprocessQstr"] = SCons.Builder.Builder(
@@ -25,8 +30,9 @@ def generate(env):
     )
 
     env["BUILDERS"]["CollectModules"] = SCons.Builder.Builder(
-        action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES"
-        " | $PYTHON $MODULECOL > $TARGET"
+        action="grep ^MP_REGISTER_MODULE $SOURCES > $TARGET"
+        # action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES"
+        # " | $PYTHON $MODULECOL > $TARGET"
     )
 
     def generate_frozen_module(source, target, env, for_signature):