libMityBuild

#!/usr/bin/env tclsh

# MityBuild
# Copyright 2018 Mark Hubbard, a.k.a. "TheMarkitecht"
# http://www.TheMarkitecht.com
#
# Project home:  http://github.com/The-Markitecht/MityBuild
# MityBuild is a small, simple project builder tool that still provides
# impressive power, ease, flexibility, and control.
#
# This file is part of MityBuild.
#
# MityBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# MityBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with MityBuild.  If not, see <https://www.gnu.org/licenses/>.


# to get started with some of the features (such as language shorthand):
#     package require MityBuild
# trouble finding that e.g. on Windows?   put:
#     TCLLIBPATH=path/to/mitybuild/repo
# prior to name of build script when invoking.  or add it to your shell environment.

package require Tcl 8.6

# ########### FILESYSTEM, OS, AND UTILITY ROUTINES ###############

interp alias {} e {} expr

proc assert {exp {msg {}}} {
    tracer asserts "testing: $exp"
    if {$msg eq {}} {set msg "assert failed: $exp"}
    set result [uplevel 1 "expr {$exp}"]
    if $result {} else {error $msg}
}

proc f+ {args} {
    return [file join {*}$args]
}

proc tracer {topic message} {
    # output the given diagnostic message if the given MityBuild trace topic is enabled.
    # note: this is totally distinct from the Tcl built-in 'trace' command.
    if { ! [tracing $topic]} return
    puts stderr "MityBuild: $message"
}

proc tracing {topic} {
    return [e {$topic in $::traceTopics}]
}
set ::traceTopics [list]

proc traceBuilt {channel} {
    puts $channel "MityBuild: built recipes:\n    [join [lsort [dict keys $::built]] "\n    "]"
}

proc traceCalls {channel} {
    puts $channel "MityBuild: recipe call tree:"
    traceCallsRecurse $channel 0 {}
}

proc traceCallsRecurse {channel callSerial indent} {
    detail = [::callTree @ $callSerial]
    prText = {}
    if [dict exists $detail run] {
        prText = " *ran*"
    }
    if [dict exists $detail provide] {
        foreach pr [detail @ provide] {
            if {$pr ne [detail @ name]} {
                append prText "\n$indent  pr:$pr"
            }
        }
    }
    puts $channel "$indent$callSerial:[detail @ name]$prText"
    if [dict exists $detail calls] {
        foreach cs [detail @ calls] {
            traceCallsRecurse $channel $cs "  $indent"
        }
    }
}

proc debugExecute {cmd args} {
    # print diagnostic output for execution debugging.
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!
    catch {
        set name [lindex [info level -1] 0]
        if {$name in $::debugProcs || {*} in $::debugProcs} {
            puts stderr ">> in $name: [string range $cmd 0 127]"
        }
    }
}

proc startDebug {args} {
    # turn on tracing of each line of code that runs in any of the given procs.
    # if one of those is * then all procs are debugged.
    set ::debugProcs $args
    trace add execution runMityBuild enterstep debugExecute
}

proc x {args} {
    # execute the given tool process with the given options.  return after it has exited.
    # return from the routine any output that wasn't redirected to stdout.
    # note that redirecting to stdout is the default in this routine.
    # to override that, specify one or more redirects along with the tool options.

    # if the first argument begins with a dash, it's assumed to be one of the following
    # error handling modes.  these determine the action if the child fails to launch
    # e.g. because the specified executable is not found, or if the child exits with
    # a non-zero exit status.
    # -abort means abort the build, provide troubleshooting messages on stderr,
    # and exit the Tcl interpreter (status 1).  this is the default action.
    # it prevents displaying (or saving to disk) a large Tcl stack trace simply because a tool failed.
    # -throw means throw a Tcl error with a descriptive message, which can be trapped by
    # try or catch.
    # -ignore means do nothing.

    # in all cases, variable childExitStatus is set in the caller's stack frame.
    # it is an integer if it could be determined from the child, otherwise empty string
    # e.g. because the child failed to launch, or was killed by a kernel signal.

    #TODO: find out why this throws an error in some environments (Geany, SystemD) when << redirection is used.

    errorAction = [args ^ 0]
    if [string match -* $errorAction] {
        if {$errorAction ni [list -abort -throw -ignore]} {
            error "Unrecognized option: $errorAction"
        }
        args = [args ^ 1 ^ end]
    } else {
        errorAction = -abort
    }
    tracer commands "Run in [pwd]:\n    $args" ;# output verbose format.
    showScript $args
    if {[lsearch -regexp $args ^>|^2>|^< ] < 0} {
        # found no argument starting with > or 2> or < chars.  add 3 standard i/o redirections.
        lappend args <@stdin >@stdout 2>@stderr
    }
    upvar childExitStatus childExitStatus
    childExitStatus = {}
    output = {}
    try {
        output = [exec -ignorestderr {*}$args ]
        childExitStatus = 0
    } trap {} {errText errDict} {
        # after a child exits with nonzero process status, this line recovers its output,
        # to be returned to the caller down below.  unfortunately it also includes some
        # explanation of the failure appended to the end by Tcl.  eliminating that would require
        # reworking this proc to redirect all output into a Tcl channel to be flushed and captured into a variable,
        # except any output that was already explicitly redirected.
        output = $errText
        # dict for {k v} $errDict { puts "<k<$k>v>$v" }
        # puts "##[errDict @ -errorinfo]##"
        sm = {}
        if [dict exists $errDict -errorcode] {
            lassign [errDict @ -errorcode] detectionMethod pid ces
            if {$detectionMethod eq {CHILDSTATUS}} {
                childExitStatus = $ces
                sm = " with exit status $ces"
            }
        }
        if {$errorAction eq {-abort}} {
            puts stderr "MityBuild: Toolchain program failed$sm:\n$args\nMityBuild aborting."
            if [tracing calls] {traceCalls stderr}
            exit 1
        } elseif {$errorAction eq {-throw}} {
            error "Toolchain program failed$sm: $args"
        }
    }
    return $output
}

proc quoteForBash {argList} {
    # pass a Tcl list.
    # return a string containing one bash shell word for each element in the given list.
    # each word is quoted as necessary for reliable use in bash shell.
    all = {}
    delim = {}
    foreach a $argList {
        if { [regexp  ^>|^2>|^<  $a] && ($delim ne {}) } {
            # found a bash redirect operator; use word as-is.
            append all "$delim$a"
        } elseif {[regexp -nocase -expanded { [^a-z0-9/._-] } $a]} {
            # surround with double quotes, first removing any existing surrounding ones,
            # and escape any in the middle with a backslash.
            append all "$delim\"[string map [list \" \\\" ] [string trim $a \" ]]\""
        } else {
            # no bash-sensitive characters; use word as-is.
            append all "$delim$a"
        }
        delim = { }
    }
    return $all
}

proc showScript {argList} {
    # output a "scriptable" copy of the given command, suitable for pasting into a shell.
    #TODO: also capture file copy, move, mkdir, delete, etc. this way.
    if [tracing script] {
        puts stderr [quoteForBash [list cd [pwd]]]
        puts stderr "    [quoteForBash $argList]"
    }
}

clock format 0 ;# make sure clock packages get loaded before redefining ::unknown; their loading depends on it.
proc ::unknown {args} {
    # this routine resolves commands that the Tcl interpreter doesn't recognize on its own.
    # it uses that hook to implement several kinds of shorthand Tcl syntax.
    # see all the comments below for the various shorthand syntax's supported.
    # it also can often recognize when the MityBuild command 'require' was forgotten.
    # the shorthand incurs a performance penalty typically 5x to 10x, partly because the
    # interp can't JIT compile that code.  so don't use it wherever fast code is needed.
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!

    #puts $args

    # bite 2 args off the front of the arg list.  repeat as long as that
    # constitutes another indexing operation for a dictionary or list.
    set assignOp {}
    set sliceType {}
    set indices {}
    set numArgs [llength $args]
    set maxArg [expr {$numArgs - 1}]
    for {set consumed 1} {$consumed < $maxArg} {} {
        set op [lindex $args $consumed]
        set idx [lindex $args ${consumed}+1]
        if {$op in {= :=}} {
            set assignOp $op
            incr consumed
            break
        } elseif {$op in {@ ^}} {
            if {($sliceType ne {}) && ($sliceType ne $op)} {
                error "More than one kind of indexing operator was used: [string range $args 0 50]"
            }
            set sliceType $op
            lappend indices $idx
            incr consumed 2
        } else {
            break
        }
    }

    # evaluate & memorize the right-hand side of an assignment.
    if {$assignOp eq {=}} {
        # infix = means assign a value, like an ordinary Tcl [set].
        if {$numArgs <= $consumed} {
            error "Did not specify a value to assign: [string range $args 0 50]"
        }
        if {$numArgs > ($consumed + 1)} {
            error "Extra words in assignment: [string range $args 0 50]"
        }
        set value [lindex $args $consumed]
    } elseif {$assignOp eq {:=}} {
        # infix := means assign from an expression.
        if {$numArgs <= $consumed} {
            error "Did not specify an expression to assign: [string range $args 0 50]"
        }
        set value [uplevel 1 [list expr [lrange $args $consumed end]]]
        #puts "expr: ~[lrange $args $consumed end]~ = $value"
    }

    # check for accidental dollar sign.
    lassign $args name
    if {[llength $name] != 1} {
        error "Variable name contains more than one word; did you place a dollar sign in front of it when you shouldn't?"
    }

    # do all kinds of assignments.
    upvar $name var
    if {$assignOp ne {}} {
        if {$sliceType eq {^}} {
            lassign $indices start end more
            if {$more ne {}} {
                error "Too many indices in list assignment: [string range $args 0 50]"
            }
            if {$end eq {}} {
                # name ^ idx means replace one list element.  name must be an ordinary variable name.
                return [lset var $start $value]]
            }
            # name ^ start ^ end means fetch from list range.  name must be an ordinary variable name, not an array or its element.
            return [set var [lreplace $var $start $end $value]]
        }
        if {$sliceType eq {@}} {
            # assign to name @ idx @ idx @ idx addressing a (possibly nested) dictionary.
            return [dict set var {*}$indices $value]
        }
        if {[string first ( $name] != -1} {
            # assign to array element.
            return [uplevel 1 [list set $name $value]]
        }
        # assign to plain variable.
        return [set var $value]
    }

    # do dictionary fetch.
    if {$sliceType eq {@}} {
        # name @ idx @ idx @ idx means fetch from (possibly nested) dictionary.
        if { ! [info exists var]} {
            error "Dictionary variable was not found: $name"
        }
        if {$numArgs > ($consumed + 1)} {
            error "Extra words in dictionary fetch: [string range $args 0 50]"
        }
        if {($numArgs > $consumed) && ! [dict exists $var {*}$indices]} {
            # an argument is given also.  that's the default value if none found.
            return [lindex $args $consumed]
        } else {
            return [dict get $var {*}$indices]
        }
    }

    # do list fetch.
    if {$sliceType eq {^}} {
        if { ! [info exists var]} {
            error "List variable was not found: $name"
        }
        if {$numArgs > $consumed} {
            error "Extra words in list fetch: [string range $args 0 50]"
        }
        lassign $indices start end more
        if {$more ne {}} {
            error "Too many indices in list fetch: [string range $args 0 50]"
        }
        if {$end eq {}} {
            # name ^ idx means fetch from list element.  name must be an ordinary variable name.
            return [lindex $var $start]
        } else {
            # name ^ start ^ end means fetch from list range.  name must be an ordinary variable name, not an array or its element.
            return [lrange $var $start $end]
        }
    }

    error "Invalid command name '$name'.  Did you mean to call a MityBuild recipe with 'require $name'?"
}

proc formatOptions {pattern  arg_list} {
    # return a space-separated string containing each item in the given arg_list formatted by the given pattern.
    # this is good e.g. for passing a whole list of paths to gcc with the -I or -L options.
    # if the arg_list contains disk paths, it's good practice to specify quote marks in the pattern to enclose them.
    # that allows for paths that include spaces etc.  but when invoking gcc, the caller must make sure the quotes
    # wrap around the entire argument.  wrapping them around just the directory path instead
    # causes gcc to see them.  then it silently uses the wrong search path (one with quotes in it) and usually fails.

    s = {}
    foreach a $arg_list {
        append s [format $pattern $a] " "
    }
    return $s
}

proc tempFilename {} {
    # each call returns a new unique file name (with absolute path) suitable to write a temporary file.
    after 1
    dir = [findImportLocal tempFileDir tempFilename:]
    return [f+ $dir "MityBuild-[clock microseconds].tmp"]
}

proc isNewer {objectFn sourceFns} {
    # returns 1 if the given objectFn is newer than every file in the given sourceFns list,
    # AND the objectFn exists, AND the sourceFns list is non-empty.  else 0.
    tracer newer "is $objectFn newer than these [llength $sourceFns] sources:\n    [join $sourceFns "\n    "]"
    if {[llength $sourceFns] == 0} {return 0}
    if { ! [file readable $objectFn]} {
        tracer newer "not found: $objectFn"
        return 0
    }
    objTime = [file mtime $objectFn]
    foreach src $sourceFns {
        if {[file exists $src] && [file mtime $src] > $objTime}  {
            tracer newer "newer source: $src"
            return 0
        }
    }
    tracer newer "newer object: $objectFn"
    return 1
}

proc recurseDepthFirst {dir filePattern dirPattern script} {
    tracer recurse "recurseDepthFirst $dir $filePattern $dirPattern"
    importLocals
    dirs = [glob -nocomplain -types d -join -- $dir $dirPattern]
    foreach child [lsort $dirs] {
        recurseDepthFirst $child $filePattern $dirPattern $script
    }
    files = [glob -nocomplain -types f -join -- $dir $filePattern]
    foreach fn [lsort $files] $script
}

proc recurseBreadthFirst {dir filePattern dirPattern script} {
    tracer recurse "recurseBreadthFirst $dir $filePattern $dirPattern $dir"
    importLocals
    files = [glob -nocomplain -types f -join -- $dir $filePattern]
    foreach fn [lsort $files] $script
    dirs = [glob -nocomplain -types d -join -- $dir $dirPattern]
    foreach child [lsort $dirs] {
        recurseBreadthFirst $child $filePattern $dirPattern $script
    }
}

proc notHiddenPattern {} {
    # returns a filename pattern that skips hidden files and dirs (those that start with a dot).
    return {[a-zA-Z0-9]*}
}

proc findFiles {filePatternsList dirPattern dir} {
    allFns = [list]
    trapExports allFns ;# in case the caller was also using this name for a local, it should remain unaffected.
    foreach filePattern $filePatternsList {
        recurseBreadthFirst  $dir  $filePattern  $dirPattern  {
            tracer recurse "found $fn"
            export+ allFns $fn
        }
    }
    return $allFns
}

proc remainder {fn root} {
    # identify and ignore the given prefix 'root' on front of the given filename 'fn'.  return the remaining portion.
    # this function is as lenient as possible; it resolves any relative paths,
    # symlinks, and extraneous dots and slashes.  relative paths are resolved from the current working dir.
    # it will throw an error if the file truly falls outside the given root dir.
    fn = [file normalize [f+ [pwd] $fn]]
    parts = [file split $fn]
    root = [file normalize [f+ [pwd] $root]]
    rootParts = [file split $root]
    if {[parts ^ 0 ^ [llength $rootParts]-1] ne $rootParts} {
        error "File '$fn' falls outside of root dir '$root'."
    }
    if {[llength $parts] == [llength $rootParts]} {return {}}
    return [f+ {*}[parts ^ [llength $rootParts] ^ end]]
}

# ########### RECIPE PARADIGM PLUMBING ROUTINES ###############

proc runMityBuild {projectInitRecipe_  projectFinalRecipe_  defaultToolchain_  args} {
    # this is the builder's main routine, suitable for parsing the command line and driving the build as a whole.
    # when calling, pass the name of the recipe to aways call first, to set up project settings etc.
    # also pass the recipe to always call last, for any odd work to wrap up the build.
    # also pass the recipe (followed by any arguments it requires) for the default toolchain for the build.

    try {
        # show help text.
        if {[llength $::commandLineRecipes] == 0} {
            ::commandLineRecipes = [dict keys $::recipes]
        }
        if {[llength $::argv] == 0} {
            puts stderr "You must specify one or more of the project's recipes:\n    [lsort $::commandLineRecipes]"
            puts stderr "\nTo call a recipe with parameters, quote the entire call with shell quote marks."
            return
        }

        # init builder.
        # now this is done at the bottom of this file instead, to better support calling
        # library functions such as 'require trace' prior to runMityBuild.

        # memorize the call to runMityBuild, for troubleshooting purposes.
        set callSerial_ [incr ::prevCallSerial]
        dict set ::callTree $callSerial_ name runMityBuild
        upvar 1 callSerial_ parentCallSerial
        dict update ::callTree $parentCallSerial detail {
            dict lappend detail calls $callSerial_
        }

        # make sure all toolchain-related variables are exported to here, so they carry to every recipe.
        require toolsCcDefault
        # defaultToolchain_ is the default toolchain; it doesn't have to be mentioned on the command line.
        # if one is selected there, it will override the defaultToolchain_ set here.
        tools = $defaultToolchain_
        assert {$tools ne {}}

        # take each command line parameter as a complete call to a recipe, or to an ordinary Tcl proc.
        # if the parameter contains multiple words, the 2nd and further words are taken as parameters to the call.
        # as a special case, 'trace' or 'tools' recipes are allowed to execute even before project init recipes.
        # the local variables here all end with an underscore so they won't import to the project's recipes.
        # however, any other locals that are exported to here by the recipes will be imported into subsequent recipes as usual.
        cd $::builderDir ;# cd into the script's own directory, in case it was executed from elsewhere.
        initDone_ = 0
        foreach cmd_ $::argv {
            name_ = [cmd_ ^ 0]
            if {$name_ ni {trace tools} && $initDone_ eq 0} {
                if {$projectInitRecipe_ ne {}} {require $projectInitRecipe_}
                require tools$tools {*}$args ;# in case the projectInitRecipe_ didn't call the selected toolchain already.
                initDone_ = 1
            }
            if [dict exists $::recipes $name_] {
                require {*}$cmd_
            } elseif {[info commands $name_] ne {}} {
                {*}$cmd_
            } else {
                return [puts stderr "MityBuild: Aborting build: Command '$name_' is not recognized as an ordinary proc, nor any of the project's recipes:\n[lsort [dict keys $::recipes]]"]
            }
        }
        if {$projectFinalRecipe_ ne {}} {require $projectFinalRecipe_}
        if [tracing calls] {traceCalls stderr}

    } trap {} {err errDict} {
        puts stderr "MityBuild: Aborting build: [lindex [split $err \n] 0]"
        stackText = "$err
\n=== STACK ===
Reminder: subtract 6 from all recipe line numbers shown.
[errDict @ -errorinfo]
\n=== ARGUMENTS ==="
        foreach {k v} [errDict @ -errorstack] {
            append stackText "\n$k: $v"
        }
        tracer builderStack $stackText
        stackFn = [f+ $::builderDir MityBuild-stack-trace.txt]
        try {
            f = [open $stackFn w]
            puts $f "$stackText\n\n=== BUILT RECIPES ==="
            traceBuilt $f
            puts $f "\n=== RECIPE CALL TREE ==="
            traceCalls $f
            close $f
            puts stderr "\nMityBuild: Builder's complete stack trace and other info saved to $stackFn"
        } trap {} e2 {
            puts stderr "\nMityBuild: Tried and failed to write stack trace to $stackFn"
        }
        puts stderr {MityBuild: Specify "trace builderStack" to also see the stack on the console.}
        exit 1
    }
}

proc recipe! {name argList exportBody runWord runBody} {
    # this procedure defines the given recipe, and also exposes it as a suggestion in the command-line help text.
    # all recipes are supported on the command line, but this keeps the help text small enough to be useful.
    recipe $name $argList $exportBody $runWord $runBody
    lappend ::commandLineRecipes $name
}

proc recipe {name argList exportBody runWord runBody} {
    # this procedure defines the given recipe.

    if {$runWord != {run}} {
        error "Syntax error: expected 'run' keyword in recipe $name"
    }
    foreach arg $argList {
        if {[llength $arg] > 1} {
            error "Syntax error: recipe $name declares an optional parameter; those are not supported."
        }
    }
    dict set ::recipes $name $argList
    # define a proc with a body combined and enhanced from the 2 given bodies of the recipe.
    # skip the runBody (the second body) if that's already executed during this build,
    # or if the caller requested to skip it by enableRunBody == 0.
    set fetchEachArg [formatOptions {$%s} $argList]
    finalArgList = [list enableRunBody {*}$argList]
    proc recipe_$name $finalArgList "
        upvar 1 callSerial_ callSerial_
        importLocals
        $exportBody
        if { \$enableRunBody } {
            if { \$enableRunBody == 2 | ! \[check $name $fetchEachArg\]} {
                dict set ::callTree \$callSerial_ run 1
                $runBody
            }
            provide {$name} $fetchEachArg
        }
    "
}
::recipes = [dict create] ;# initialize before any recipes are defined in this library; else they would be lost.
::commandLineRecipes = [list]

proc provide {args} {
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!
    tracer depends "provide [string range $args 0 255]"
    upvar 1 callSerial_ callSerial_
    dict update ::callTree $callSerial_ detail {
        dict lappend detail provide $args
    }
    dict incr ::built $args
}

proc requireCore {name enableRunBody argsList} {
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!
    tracer depends "require $name [string range $argsList 0 255]"
    # memorize the call for troubleshooting purposes.
    set callSerial_ [incr ::prevCallSerial]
    dict set ::callTree $callSerial_ name $name
    upvar 1 callSerial_ parentCallSerial
    dict update ::callTree $parentCallSerial detail {
        dict lappend detail calls $callSerial_
    }
    tracer calls "call $callSerial_ to $name from [dict get $::callTree $parentCallSerial name]"
    set pwd_ [pwd]
    set deferExports_ [list]
    recipe_$name $enableRunBody {*}$argsList
    foreach cmd $deferExports_ {
        {*}$cmd
    }
    cd $pwd_ ;# restore the previous working directory after each 'require'.
}

proc require {name args} {
    # 'require' places a call to a recipe.  they can't be called by their name alone,
    # and they require special handling during calls.  that's implemented here.
    uplevel 1 [list requireCore $name 1 $args]
}

proc requireExports {name args} {
    # 'requireExports' is identical to 'require' except only the recipe's exportBody is
    # used.  its runBody is ignored during this call.  this is useful e.g. when
    # a recipe for a library exports all of its paths, and another recipe needs to re-use
    # those paths in order to clean that library.
    uplevel 1 [list requireCore $name 0 $args]
}

proc force {name args} {
    # 'force' is identical to 'require' except the recipe's runBody will
    # definitely execute.  this is useful when you need to explicitly re-run
    # a recipe with the same parameters as before.

    uplevel 1 [list requireCore $name 2 $args]
}

proc check {args} {
    # returns true if the given recipe has already been called with the given arguments during this build.
    c = [dict exists $::built $args]
    if [dict exists $::built checks] { ;# can't call 'tracer' here because it relies on 'check'.
        puts stderr "MityBuild: check=$c:[string range $args 0 255]"
    }
    return $c
}
::built = [dict create]

proc forgetBuilt {args} {
    # forgets having ever built the specified recipe (with arguments) already.
    # if no recipe is given, forgets ALL built recipes.
    # this is useful when you need to explicitly re-run one or several
    # recipes with the same parameters as before.

    if {[llength $args]} {
        ::built = [dict remove $::built $args]
    } else {
        ::built = [dict create]
    }
}

proc importLocals {} {
    # propagate values of all local variables (except arrays) into the current stack frame from its caller
    # (the most-recently-started recipe).  locals whose names end in an underscore are ignored.
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!
    set procName [lindex [info level -1] 0]
    set procParms [info args $procName]
    tracer imports "importLocals to $procName:"
    # ignore 'require' and other frames that interpose between recipe frames.  find the nearest recipe frame.
    set fromFrame -1
    for {set i [expr [info level] - 2]}  {$i >= 0}  {incr i -1} {
        set fromProcName [lindex [info level $i] 0]
        if {[string match recipe_* $fromProcName] || $fromProcName eq {runMityBuild} || ! [string match recipe_* $procName]} {
            set fromFrame $i
            break
        }
    }
    if {$fromFrame == -1} return ;# no recipe frames were found on the stack.
    tracer imports "    from $fromProcName:"
    foreach name [uplevel #$fromFrame {info vars}] {
        if [string match *_ $name] continue
        if {$name in $procParms || $name in $::ignoreImports} continue
        if [uplevel #$fromFrame [list array exists $name]] continue
        upvar #$fromFrame $name vFrom
        upvar 1 $name v1
        set v1 $vFrom
        tracer imports "    $name=$vFrom"
    }
}

proc findImportLocal {name {errorMsg {noError}}} {
    # walk up the call stack until a local variable (not array) with the given name is found,
    # and import its value as a local in the current proc.  also return the value.
    # if it's never found, the given error message is thrown.  that's optional.
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!
    for {set i [expr [info level] - 2]}  {$i >= 0}  {incr i -1} {
        if { ! [uplevel #$i [list info exists $name]]} continue
        if [uplevel #$i [list array exists $name]] continue
        upvar #$i $name found
        upvar 1 $name v1
        set v1 $found
        tracer imports "findImportLocal $name=$found"
        return $found
    }
    if {$errorMsg ne {noError}} {
        error "$msg\nfindImportLocal: could not find variable '$name'"
    }
}

proc exportCore {name op} {
    # apply the given operation script to the same-named variable all the way up the call stack,
    # until reaching a stack frame where the var doesn't exist, or the var is an array (incompatible),
    # or the var is in the local trapVars_ list.  apply to a minimum of 2 frames (assuming no traps):
    # the current proc (which called export, not exportCore), and its caller.
    # the script must operate on a variable named 'var'.
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!
    set numDone 0
    for {set i [expr [info level] - 2]}  {$i >= 0}  {incr i -1} {
        tracer exportsMore "done $numDone i $i: [info level $i]"
        set procName [lindex [info level $i] 0]
        set level #$i
        if {$procName in {requireCore require requireExports}} continue ;# ignore 'require' frames that always interpose between recipe frames.
        if {$numDone >= 2 && ! [uplevel $level [list info exists $name]]} break
        if [uplevel $level [list array exists $name]] break
        upvar $level $name var
        eval $op
        incr numDone
        tracer exports "export to $procName at level $i: [string range [string trim $op] 0 2] $name = [string range [string trim $var] 0 63]"
        upvar $level trapVars_ traps$i
        if {[info exists traps$i] && $name in [set traps$i]} break
    }
}

proc export {name value} {
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!
    exportCore $name [list set var $value]
}

proc export+ {name args} {
    # DO NOT USE SHORTHAND TCL IN THIS ROUTINE!
    if {[llength $args] == 0} {
        exportCore $name [list lappend var] ;# important side effect is to create the list if it doesn't exist.
    } else {
        exportCore $name "
            lappend var ;# important side effect is to create the list if it doesn't exist.
            foreach value [list $args] {
                if {\$value ni \$var} {
                    lappend var \$value
                }
            }"
    }
}

proc trapExports {varNameList} {
    # prevent any of the variable names in the given list from being exported up to the
    # current recipe from any called recipes.  for example, a distinct toolchain can be
    # isolated from all other recipes by 'trapExports $toolVars' then 'require myTools'.
    upvar trapVars_ trapVars_
    lappend trapVars_ {*}$varNameList
}

proc defer {args} {
    # memorize the given command, to apply later, after the end of the recipe.
    upvar 2 deferExports_ exp
    lappend exp $args
}

############### REUSABLE RECIPES FOR TYPICAL PROJECTS #####################

recipe! trace {args} {} run {
    # recipe for the command line to activate a tracer topic.
    # note: this is totally distinct from the Tcl built-in 'trace' command.
    lappend ::traceTopics {*}$args
}

recipe! tools {toolchain} {
    # recipe to provide easy 'tools' word syntax on command line.
    # also important for delaying initialization of the toolchain until the project recipes are ready for that.
    if { ! [dict exists $::recipes tools$toolchain]} {
        t = [lsort [dict keys $::recipes tools*]]
        t = [lmap r $t {string range $r 5 end}]
        t = [t ^ 1 ^ end] ;# remove the empty item resulting from the 'tools' recipe; it's always first after sorting.
        error "Requested toolchain '$toolchain' was not found among the project's toolchain recipes (case sensitive): $t"
    }
    export tools $toolchain
} run {}

recipe compile {objectFn sourceFn} {} run {
    require $toolsCompilerRecipe $objectFn $sourceFn
}

recipe linkExecutable {binFn} {} run {
    require $toolsLinkerRecipe $binFn
}

recipe linkDynamicLib {binFn} {} run {
    require $toolsDynamicLinkerRecipe $binFn
}

recipe compileAndLink {binFn linkType sourceRootDir sourceFns} {} run {
    # compile each file of the given sourceFns into an object file (.o) with the same name,
    # and link those into a binary using the given linkType recipe.
    # linkType might be e.g. linkExecutable or linkDynamicLib or packStaticLib.
    # *.o and other intermediate files are built in a deeper subtree around binFn,
    # that mirrors the source tree under the given sourceRootDir.
    linkerObjFns = [list]
    foreach source $sourceFns {
        # invoke compiler recipe once for this source file alone, accumulating a list of linkerObjFns.
        objBare = [file rootname [file tail $source]].o
        childDir = [remainder [file dirname $source] $sourceRootDir]
        objFull = [f+ [file dirname $binFn] $childDir $objBare]
        require compile $objFull $source
    }
    # link all linkerObjFns together into a binary.
    require $linkType $binFn
}

recipe packStaticLib {binFn} {} run {
    # static libs are actually packed into 1 file by the
    # archiver tool rather than actually linked with the linker.
    require $toolsPackStaticLibRecipe $binFn
}

proc exportLibrary {libName {linkType packStaticLib} {libSrcDir {}}} {
    # this routine exports the usual variables for a library.
    # those are named after the given library name.
    # using this is optional; it just avoids some boilerplate export's and file join's in cases where the
    # names and paths are sufficiently predictable.
    # an ordinary recipe for a specific library would be much simpler;
    # the extra complexity is here because it's generic.  it's generating recipe code on the fly.

    importLocals
    if {$libSrcDir eq {}} {
        # assume directory name is same as libName
        libSrcDir = [f+ $projectSrcDir $libName]
    }
    binDir = [f+ $projectBinDir [remainder $libSrcDir $projectSrcDir]]
    if {[file tail $binDir] ne $libName} {
        binDir = [f+ $binDir $libName]
    }
    libSearchCode = {}
    if {$linkType eq {packStaticLib}}  {
        binFn = [f+ $binDir lib$libName.a]
        libList = staticLibFns
    } else {
        binFn = [f+ $binDir lib$libName.so]
        libList = dynamicLibNames
        libSearchCode = "export+ libSearchDirs {$binDir}"
    }
    code = "
        export ${libName}SrcDir {$libSrcDir}
        export ${libName}BinDir {$binDir}
        export ${libName}BinFn {$binFn}
        export+ includeDirs {$libSrcDir}
        $libSearchCode
        defer export+ $libList {$binFn}
    "
    tracer exportLibrary $code
    uplevel 1 $code
}

############### REUSABLE LOW-LEVEL RECIPES FOR CC-STYLE TOOLCHAINS #####################
# ...such as gcc, and often clang.

recipe toolsCcDefault {} {
    export cc cc  ;# C compiler program name.
    export ld $cc  ;# use same tool again for linking.
    export ar ar ;# program name of archiver for static libraries.
    export readelf readelf ;# program name of ELF binary file parser.
    # optimization; s = small code, 0 = no optim (best for source-level debugging).
    if { ! [info exists optim]} {export optim 0}
    if { ! [info exists debugLevel]} {export debugLevel 3}
    export commonOptions [list] ;# used for all compiler and linker calls.
    export sourceDependencyOptions [list] ;# causes dumping source file dependency list into .d file.
    export sourceLanguagePatterns [dict create \
        *.c*  cLanguageOptions  \
        *.s  asmLanguageOptions  \
        *.asm  asmLanguageOptions ]
    export cLanguageOptions [list {*}$commonOptions -c {*}$sourceDependencyOptions] ;# options to compile only, without linking.
    export asmLanguageOptions [list {*}$commonOptions -c -xassembler-with-cpp {*}$sourceDependencyOptions] ;# options to assemble only, without linking.
    export linkerOptions [list {*}$commonOptions] ;# options to pass to linker.
    export libOptions [list] ;# extra options used only for building a static or dynamic library.
    export includeDirs [list] ;# list of all directories to search for #include files, in the order they appear here.
    export staticLibFns [list] ;# list of all static library filenames (with paths) to include in a link, in the order they appear here.
    export dynamicLibNames [list] ;# list of bare (undecorated) names of all dynamic (shared) libraries to include in a link, in the order they appear here.
        # for example, pthread here would link to libpthread.so in whatever path it can be found.
    export libSearchDirs [list] ;# list of all directories to search for static or dynamic library files, in the order they appear here.
    export toolsCompilerRecipe ccCompile ;# name of recipe that can build command line options and invoke cc program (C compiler).
        # OK to use cc-style command recipe as a default; many toolchains accept that style of command line options.
    export toolsLinkerRecipe ccLink ;# name of recipe that can build command line options and invoke ld program (linker).
    # toolsLinkerRecipe is for linking standalone executables.  these next 2 are similar, for dynamic and static libraries instead.
    export toolsDynamicLinkerRecipe ccDynamicLink
    export toolsPackStaticLibRecipe ccPackStaticLib
    export toolsGetSourceFileDependenciesProc ccGetSourceFileDependencies
        # name of ordinary procedure (not recipe) to fetch source file dependencies from the compiler.  see ccGetSourceFileDependencies for details.
        # OK to use cc-style command recipe as a default; it works with both gcc and clang, and maybe others.
    export+ defineSymbols ;# list of all C preprocessor symbols to be passed to cc (C compiler), in the order they appear here.
        # for example, BUILDME=YES here would cause -DBUILDME=YES to be passed to cc.
        # any existing symbols already in this list prior to this line of code will be preserved also.
    export tempFileDir /tmp ;# directory where temporary files can be placed during the build.
    # these lists tell which variables to specify to trapExports whenever a recipe needs to trap exports of changes to its toolchain.
    # typically that would be done only if a group of recipes needs to use a different toolchain locally than the rest of the project.
    # these lists are reasonably complete, and might not need to be overridden.
    export toolVars {
        cc  ld  ar  readelf
        toolsCompilerRecipe  toolsLinkerRecipe  toolsDynamicLinkerRecipe
        toolsPackStaticLibRecipe  toolsGetSourceFileDependenciesProc }
    export toolOptionsVars {commonOptions sourceDependencyOptions sourceLanguagePatterns
        cLanguageOptions asmLanguageOptions linkerOptions libOptions defineSymbols}
    export toolLibVars {includeDirs staticLibFns dynamicLibNames libSearchDirs}
} run {}

recipe ccCompile {objectFn sourceFn} {} run {
    # run a cc-style compiler tool, passing it the local cLanguageOptions
    # and includeDirs and defineSymbols, to compile the given sourceFn
    # (and any other source dependencies such as header files) into the given objectFn.
    # asmLanguageOptions or other language options may be selected instead of cLanguageOptions,
    # by comparing the filename to the patterns in sourceLanguagePatterns dictionary.
    export+ linkerObjFns $objectFn
    deps = [$toolsGetSourceFileDependenciesProc $source $objectFn]
    if [isNewer $objectFn [concat $sourceFn $deps]] return
    languageOptions = $cLanguageOptions
    tail = [file tail $sourceFn]
    dict for {k v} $sourceLanguagePatterns {
        if [string match -nocase $k $tail] {
            languageOptions = [set $v] ;# fetch the languageOptions variable named in the dictionary.
        }
    }
    lappend languageOptions {*}[formatOptions {"-I%s"} $includeDirs]
    lappend languageOptions {*}[formatOptions {"-D%s"} $defineSymbols]
    objectDir = [file dirname $objectFn]
    file mkdir $objectDir
    file delete $objectFn ;# leave no file behind in case the tool fails.
    cd $objectDir ;# to contain any additional files the tool may write.
    x $cc  -o $objectFn  {*}$languageOptions  $sourceFn
    provide compile done $objectFn ;# recipes can check for this to see if a tool was really executed here, not skipped.
}

recipe ccPackStaticLib {binFn} {} run {
    # run the cc-style archiver tool to create a .a archive file having the given name
    # and containing the local linkerObjFns.
    # see ccLink for notes about Windows command length limits.
    if {[llength $linkerObjFns] == 0} {
        error "Empty list of object files to pack into archive $binFn"
    }
    if {[isNewer $binFn $linkerObjFns]} return
    file mkdir [file dirname $binFn]
    lsfn = [tempFilename]
    lscript = [open $lsfn w]
    tracer linker "Packing into $binFn:"
    foreach fn $linkerObjFns {
        puts $lscript $fn
        tracer linker "    $fn"
    }
    close $lscript
    file mkdir [file dirname $binFn]
    file delete $binFn ;# leave no file behind in case the tool fails.
    x $ar -src $binFn @$lsfn
    file delete $lsfn
    provide pack done $binFn ;# recipes can check for this to see if a tool was really executed here, not skipped.
}

recipe ccDynamicLink {binFn} {} run {
    # link a shared library for dynamic linking.
    lappend linkerOptions -shared
    require ccLink $binFn
    # show shared library dependencies for any executable or shared library:
    #x objdump -x $fn | grep -i needed
    # show symbols provided by library:
    #x objdump -t $fn
}

recipe ccLink {binFn} {} run {
    # run a cc-style linker tool, passing it the local linkerOptions, to link
    # the local linkerObjFns and staticLibFns and dynamicLibNames into the given binFn.

    # build list of all objects to be linked.
    # add static library archive file names last of all, to satisfy symbol dependencies.
    # a library that defines a symbol must come AFTER the objects that use it.
    # that's counter-intuitive, but that's how gcc (really ld) works.
    objAndStaticLibs = [concat $linkerObjFns $staticLibFns]
    if {[llength $objAndStaticLibs] == 0} {
        error "Empty list of object files to link into $binFn"
    }
    # avoid rerunning the link when it's not needed.
    if [isNewer $binFn $objAndStaticLibs] return
    tracer linker "Linking into $binFn:"
    # build linker command line, starting with the local linkerOptions.
    # add args particular to the linker, so those are in effect for all input object files.
    linkerOptions = [list -o $binFn  {*}$linkerOptions       \
            -Wl,-Map=[f+ [file dirname $binFn] linker.map]  \
            {*}[formatOptions {"-D%s"} $defineSymbols]]
    # add library search paths, so they're in effect for all libraries that follow.
    lappend linkerOptions  {*}[formatOptions {"-L%s"} $libSearchDirs]
    # add the list of input files.
    # THIS INCLUDES STATIC LIBRARY ARCHIVE FILE NAMES;  .a files with paths.
    # note: Windows can't cope with a command long enough to pass all the files on the command line.
    # the limit is 32767 chars for all arguments in Win32 API CreateProcess().
    # @file syntax could avoid that.  but Altera's gcc 4.1.2 doesn't support @file syntax; it was invented later.
    # a temporary shell script won't help, since the shell's limit is 8192, so it's actually worse.
    # use a temporary linker script instead.
    lsfn = [tempFilename]
    lappend linkerOptions  $lsfn
    lscript = [open $lsfn w]
    foreach fn $objAndStaticLibs {
        puts $lscript "INPUT($fn)"
        tracer linker "    $fn"
    }
    close $lscript
    # add dynamic library names last of all, to satisfy symbol dependencies.
    # a library that defines a symbol must come AFTER the objects that use it.
    # that's counter-intuitive, but that's how gcc (really ld) works.
    lappend linkerOptions  {*}[formatOptions {"-l%s"} $dynamicLibNames]
    tracer linker "    dynamic libs:\n        [join $dynamicLibNames "\n        "]"
    if [tracing linker] {
        tracer linker "Linker trace is enabled.  Linker will list out all object files it uses."
        lappend linkerOptions -Wl,-t
    }
    # make directory for output bin file, and execute linker.
    toolDir = [file dirname $binFn]
    file mkdir $toolDir
    file delete $binFn ;# leave no file behind in case the tool fails.
    cd $toolDir ;# to contain any additional files the tool may write.
    x  $ld  -Wl,--start-group  {*}$linkerOptions  -Wl,--end-group
    # ^^^ --start-group is used there as a cheap hack.  the alternative would be automatically
    # sorting all the static and dynamic libraries (and .o files?) in order of dependency.
    file delete $lsfn ;# no need to keep this file; its content can be traced above.
    export linkerObjFns [list] ;# reset this list to potentially prepare for the next link.
    provide link done $binFn ;# recipes can check for this to see if a tool was really executed here, not skipped.
}

proc ccGetSourceFileDependencies {sourceFn  objectFn} {
    # recipes using a cc-style toolchain can call this proc to fetch source file dependencies FROM THE LAST BUILD.
    # all toolchain-specific details of how that's done are encapsulated here.
    # using the cc toolchain, dependencies can be read from a .d file corresponding to the .o file.
    # in C that includes all header files that the source file #include's, and all the ones #include'd by those, etc.
    # to create the .d file, specify "-MMD -MT sources" on every call to cc.

    dependencyDumpFn = [file rootname $objectFn].d
    if { ! [file readable $dependencyDumpFn]} return [list]
    f = [open $dependencyDumpFn r]
    content = [string trim [read $f]]
    close $f
    if { ! [string match -nocase  sources:*  $content]} {
        error "Unrecognized prefix found in dependency file content:\n$depFn \nPrefix must be \"sources:\"."
    }
    content = [string trim [string range $content 8 end]]
    parsed = [list {*}$content]
    return [lmap s $parsed {string trim $s}]
}

# ########### PRACTICAL TOOLCHAIN SETUP RECIPES ###############

recipe toolsClang {} {
    # Clang toolchain settings usable for all recipes.  for clang with their libc++.
    # see recipe toolsCcDefault for explanations.
    # sudo apt-get install libc++-dev
    export cc clang++  ;# Clang for the PC host's operating system.
    export ld $cc  ;# use same tool again for linking.
    export commonOptions [list -pipe -O$optim -g$debugLevel -Wall -fmessage-length=0 -fPIC]
    export sourceDependencyOptions [list -MMD -MT sources] ;# causes dumping source file dependency list into .d file.
    export cLanguageOptions [list {*}$commonOptions -c -xc++ -std=c++14 -stdlib=libc++ {*}$sourceDependencyOptions]
    export asmLanguageOptions [list {*}$commonOptions -c -xassembler-with-cpp {*}$sourceDependencyOptions]
    export linkerOptions [list {*}$commonOptions -std=c++14 -stdlib=libc++ -pthread]
    export libOptions [list -fpic -shared] ;# -rdynamic
} run {}

recipe toolsClangGnuLib {} {
    # for clang with GNU libstdc++ instead of LLVM's libc++, which also affects examination of data structures in gdb.
    require toolsClang
    export cLanguageOptions [list {*}$commonOptions -c -xc++ -std=c++14 -stdlib=libstdc++ {*}$sourceDependencyOptions]
    export linkerOptions [list {*}$commonOptions -std=c++14 -stdlib=libstdc++ -pthread]
} run {}

recipe toolsClang11 {} {
    # for clang with their libc++, with C++11 standard.
    require toolsClang
    export cLanguageOptions [list {*}$commonOptions -c -xc++ -std=c++11 -stdlib=libc++ {*}$sourceDependencyOptions]
    export linkerOptions [list {*}$commonOptions -std=c++11 -stdlib=libc++ -pthread]
} run {}

recipe toolsClangC11 {} {
    # for clang with its default standard library, with C11 (not C++11) standard.
    require toolsClang
    export cLanguageOptions [list {*}$commonOptions -c -xc -std=c11 {*}$sourceDependencyOptions]
    export linkerOptions [list {*}$commonOptions -std=c11 -pthread]
} run {}

recipe toolsGcc {} {
    # gcc toolchain settings for all recipes.  GCC for the PC's native operating system with GNU libstdc++.
    export cc g++
    export ld $cc  ;# use same tool again for linking.
    export commonOptions [list -pipe -O$optim -g$debugLevel -Wall -fmessage-length=0 -fPIC]
    export sourceDependencyOptions [list -MMD -MT sources] ;# causes dumping source file dependency list into .d file.
    export cLanguageOptions [list {*}$commonOptions -c -xc++ -std=c++14 {*}$sourceDependencyOptions]
    export asmLanguageOptions [list {*}$commonOptions -c -xassembler-with-cpp {*}$sourceDependencyOptions]
    export linkerOptions [list {*}$commonOptions -std=c++14 -pthread]
    export libOptions [list -fpic -shared] ;# -rdynamic
} run {}

recipe toolsGccClangLib {} {
    # for gcc with clang's libc++
    require toolsGcc
    export cLanguageOptions [list {*}$commonOptions -c -xc++  \
        -std=c++14 -stdlib=libstdc++ {*}$sourceDependencyOptions  \
        -nostdinc++ -I/usr/include/c++/v1   \
        -nodefaultlibs -lc++ -lm -lc -lgcc_s -lgcc  ]
    export linkerOptions [list {*}$commonOptions -std=c++14 -stdlib=libstdc++ -pthread]
} run {}

recipe toolsGccArm {} {
    # gcc ARM cross-compile toolchain settings for all recipes.  GCC for embedded devices with Linux EABI and ARM hard-float.
    require toolsGcc
    export cc arm-linux-gnueabihf-g++
    export ld $cc  ;# use same tool again for linking.
    export ar arm-linux-gnueabihf-ar
    export readelf arm-linux-gnueabihf-readelf
} run {}

recipe toolsGccArmC11 {} {
    # gcc ARM cross-compile toolchain, with C11 (not C++11) standard.
    require toolsGccArm
    export cLanguageOptions [list {*}$commonOptions -c -xc -std=c11 {*}$sourceDependencyOptions]
    export linkerOptions [list {*}$commonOptions -std=c11 -pthread]
} run {}

recipe toolsGccArmClangLib {} {
    # for ARM cross-compile with gcc with clang's libc++;
    # see http://libcxx.llvm.org/docs/UsingLibcxx.html#using-libc-with-gcc
    require toolsGccArm
    export cLanguageOptions [list {*}$commonOptions -c -xc++  \
        -std=c++14 -stdlib=libstdc++ {*}$sourceDependencyOptions  \
        -nostdinc++ -I/usr/include/c++/v1   \
        -nodefaultlibs -lc++ -lm -lc -lgcc_s -lgcc  ]
    export linkerOptions [list {*}$commonOptions -std=c++14 -stdlib=libstdc++ -pthread]
} run {}

############### INITIALIZE LIBMITYBUILD #####################
::builderDir = [file normalize [f+ [pwd] [file dirname $::argv0]]]
if {$::tcl_interactive} {::builderDir = [pwd]}
::ignoreImports = [list args] ;# never import the variable 'args'; that just clutters the frame.
::built = [dict create]
::callTree = [dict create 0 [dict create name builderScriptFile]]
::prevCallSerial = 0
callSerial_ = 0
package provide MityBuild 3.0