diff --git a/common/tools/cointool.py b/common/tools/cointool.py index 8eb8fc1e7b1..cfef782c9e2 100755 --- a/common/tools/cointool.py +++ b/common/tools/cointool.py @@ -3,7 +3,6 @@ import datetime import fnmatch -import glob import json import logging import os @@ -11,6 +10,7 @@ import sys from collections import defaultdict from hashlib import sha256 +from pathlib import Path from typing import Any, Callable, Iterator, TextIO, cast import click @@ -133,13 +133,13 @@ def supported_on(device: str, coins: Coins) -> Iterator[Coin]: def render_file( - src: str, dst: TextIO, coins: CoinsInfo, support_info: SupportInfo + src: Path, dst: Path, coins: CoinsInfo, support_info: SupportInfo ) -> None: """Renders `src` template into `dst`. `src` is a filename, `dst` is an open file object. """ - template = mako.template.Template(filename=src) + template = mako.template.Template(filename=str(src.resolve())) eth_defs_date = datetime.datetime.fromisoformat( DEFINITIONS_TIMESTAMP_PATH.read_text().strip() ) @@ -150,7 +150,9 @@ def render_file( **coins, **MAKO_FILTERS, ) - dst.write(result) + dst.write_text(str(result)) + src_stat = src.stat() + os.utime(dst, ns=(src_stat.st_atime_ns, src_stat.st_mtime_ns)) # ====== validation functions ====== @@ -840,13 +842,13 @@ def modify_coin(coin: Coin) -> Coin: @cli.command() # fmt: off -@click.argument("paths", metavar="[path]...", nargs=-1) -@click.option("-o", "--outfile", type=click.File("w"), help="Alternate output file") +@click.argument("paths", type=click.Path(path_type=Path), metavar="[path]...", nargs=-1) +@click.option("-o", "--outfile", type=click.Path(dir_okay=False, writable=True, path_type=Path), help="Alternate output file") @click.option("-v", "--verbose", is_flag=True, help="Print rendered file names") @click.option("-b", "--bitcoin-only", is_flag=True, help="Accept only Bitcoin coins") # fmt: on def render( - paths: tuple[str, ...], outfile: TextIO, verbose: bool, bitcoin_only: bool + paths: tuple[Path, ...], outfile: Path, verbose: bool, bitcoin_only: bool ) -> None: """Generate source code from Mako templates. @@ -882,7 +884,7 @@ def render( for key, value in support_info.items(): support_info[key] = Munch(value) - def do_render(src: str, dst: TextIO) -> None: + def do_render(src: Path, dst: Path) -> None: if verbose: click.echo(f"Rendering {src} => {dst.name}") render_file(src, dst, defs, support_info) @@ -894,25 +896,23 @@ def do_render(src: str, dst: TextIO) -> None: # find files in directories if not paths: - paths = (".",) + paths = (Path(),) - files: list[str] = [] + files: list[Path] = [] for path in paths: - if not os.path.exists(path): + if not path.exists(): click.echo(f"Path {path} does not exist") - elif os.path.isdir(path): - files += glob.glob(os.path.join(path, "*.mako")) + elif path.is_dir(): + files.extend(path.glob("*.mako")) else: files.append(path) # render each file for file in files: - if not file.endswith(".mako"): + if not file.suffix == ".mako": click.echo(f"File {file} does not end with .mako") else: - target = file[: -len(".mako")] - with open(target, "w") as dst: - do_render(file, dst) + do_render(file, file.parent / file.stem) @cli.command() diff --git a/core/Makefile b/core/Makefile index a2742ea3799..6463a6ef8db 100644 --- a/core/Makefile +++ b/core/Makefile @@ -1,6 +1,12 @@ .PHONY: vendor -JOBS = 4 +# get the number of CPU cores in a "portable" manner +# (accounting for darwin and big.LITTLE archs) +JOBS = $(shell sysctl -n hw.perflevel0.physicalcpu 2>/dev/null ||\ + sysctl -n hw.physicalcpu 2>/dev/null ||\ + sysctl -n hw.ncpu 2>/dev/null ||\ + nproc 2>/dev/null ||\ + echo 4) MAKE = make -j $(JOBS) SCONS = scons -Q -j $(JOBS) diff --git a/core/SConscript.firmware b/core/SConscript.firmware index 85cd7f6c335..23d4959e727 100644 --- a/core/SConscript.firmware +++ b/core/SConscript.firmware @@ -489,8 +489,10 @@ qstr_protobuf = env.Command( qstr_micropython = 'vendor/micropython/py/qstrdefs.h' +micropy_defines = env.MicroPyDefines(source=SOURCE_QSTR) + qstr_collected = env.CollectQstr( - target='genhdr/qstrdefs.collected.h', source=SOURCE_QSTR) + target='genhdr/qstrdefs.collected.h', source=micropy_defines) qstr_preprocessed = env.PreprocessQstr( target='genhdr/qstrdefs.preprocessed.h', @@ -506,16 +508,16 @@ env.Ignore(qstr_collected, qstr_generated) # moduledefs_collected = env.CollectModules( - target='genhdr/moduledefs.collected.h', source=SOURCE_QSTR) + target='genhdr/moduledefs.collected.h', source=micropy_defines) hdr_moduledefs = env.Command( target='genhdr/moduledefs.h', source=moduledefs_collected, action='$MAKEMODULEDEFS $SOURCE > $TARGET', ) -env.Ignore(moduledefs_collected, moduledefs_collected) -env.Ignore(moduledefs_collected, qstr_generated) -env.Ignore(moduledefs_collected, hdr_moduledefs) +env.Ignore(micropy_defines, micropy_defines) +env.Ignore(micropy_defines, qstr_generated) +env.Ignore(micropy_defines, hdr_moduledefs) # # Micropython version diff --git a/core/SConscript.unix b/core/SConscript.unix index 94617ef1a0f..1f50c9733de 100644 --- a/core/SConscript.unix +++ b/core/SConscript.unix @@ -564,8 +564,10 @@ qstr_protobuf = env.Command( qstr_micropython = 'vendor/micropython/py/qstrdefs.h' +micropy_defines = env.MicroPyDefines(source=SOURCE_QSTR) + qstr_collected = env.CollectQstr( - target='genhdr/qstrdefs.collected.h', source=SOURCE_QSTR) + target='genhdr/qstrdefs.collected.h', source=micropy_defines) qstr_preprocessed = env.PreprocessQstr( target='genhdr/qstrdefs.preprocessed.h', @@ -581,7 +583,7 @@ env.Ignore(qstr_collected, qstr_generated) # moduledefs_collected = env.CollectModules( - target='genhdr/moduledefs.collected.h', source=SOURCE_QSTR) + target='genhdr/moduledefs.collected.h', source=micropy_defines) hdr_moduledefs = env.Command( @@ -602,10 +604,10 @@ hdr_version = env.Command( source='', action='$MAKEVERSIONHDR $TARGET', ) -env.Ignore(hdr_moduledefs, hdr_moduledefs) -env.Ignore(hdr_moduledefs, qstr_collected) -env.Ignore(hdr_moduledefs, qstr_preprocessed) -env.Ignore(hdr_moduledefs, qstr_generated) +env.Ignore(micropy_defines, hdr_moduledefs) +env.Ignore(micropy_defines, qstr_collected) +env.Ignore(micropy_defines, qstr_preprocessed) +env.Ignore(micropy_defines, qstr_generated) # # Frozen modules diff --git a/core/SConstruct b/core/SConstruct index a0436431341..d4c5aae455b 100644 --- a/core/SConstruct +++ b/core/SConstruct @@ -1,4 +1,7 @@ # pylint: disable=E0602 +Decider('content-timestamp') +SetOption('max_drift', 10) +SetOption('implicit_cache', 1) SConscript('SConscript.boardloader', variant_dir='build/boardloader', duplicate=False) SConscript('SConscript.bootloader', variant_dir='build/bootloader', duplicate=False) diff --git a/core/site_scons/site_tools/micropython/__init__.py b/core/site_scons/site_tools/micropython/__init__.py index 80a03ca6b48..cf231a673a3 100644 --- a/core/site_scons/site_tools/micropython/__init__.py +++ b/core/site_scons/site_tools/micropython/__init__.py @@ -8,9 +8,14 @@ def generate(env): MODULECOL="site_scons/site_tools/micropython/moduledefs.py", ) + env["BUILDERS"]["MicroPyDefines"] = SCons.Builder.Builder( + action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCE > $TARGET", + suffix=".upydef", + single_source=True, + ) + env["BUILDERS"]["CollectQstr"] = SCons.Builder.Builder( - action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES" - " | $PYTHON $QSTRCOL > $TARGET" + action="cat $SOURCES | perl -nle 'print \"Q($1)\" while /MP_QSTR_(\\w+)/g' > $TARGET" ) env["BUILDERS"]["PreprocessQstr"] = SCons.Builder.Builder( @@ -25,8 +30,9 @@ def generate(env): ) env["BUILDERS"]["CollectModules"] = SCons.Builder.Builder( - action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES" - " | $PYTHON $MODULECOL > $TARGET" + action="grep ^MP_REGISTER_MODULE $SOURCES > $TARGET" + # action="$CC -E $CCFLAGS_QSTR $CFLAGS $CCFLAGS $_CCCOMCOM $SOURCES" + # " | $PYTHON $MODULECOL > $TARGET" ) def generate_frozen_module(source, target, env, for_signature): diff --git a/core/site_scons/site_tools/micropython/moduledefs.py b/core/site_scons/site_tools/micropython/moduledefs.py deleted file mode 100644 index d6efb3ca467..00000000000 --- a/core/site_scons/site_tools/micropython/moduledefs.py +++ /dev/null @@ -1,13 +0,0 @@ -import re -import sys - - -def process(source, target): - re_module = re.compile(r"MP_REGISTER_MODULE\(.*?,\s*.*?\);") - for line in source: - for match in re_module.findall(line): - target.write(f"{match}\n") - - -if __name__ == "__main__": - process(sys.stdin, sys.stdout) diff --git a/core/site_scons/site_tools/micropython/qstrdefs.py b/core/site_scons/site_tools/micropython/qstrdefs.py deleted file mode 100644 index 718935989a2..00000000000 --- a/core/site_scons/site_tools/micropython/qstrdefs.py +++ /dev/null @@ -1,14 +0,0 @@ -import re -import sys - - -def process(source, target): - re_qstr = re.compile(r"MP_QSTR_[_a-zA-Z0-9]+") - for line in source: - for match in re_qstr.findall(line): - name = match.replace("MP_QSTR_", "") - target.write(f"Q({name})\n") - - -if __name__ == "__main__": - process(sys.stdin, sys.stdout)