From 7c90e20885cb53d2b3319e9fdb93a80e83cf95bd Mon Sep 17 00:00:00 2001 From: Eryk Szpotanski Date: Fri, 20 Sep 2024 20:02:56 +0200 Subject: [PATCH] tools: AutoTuner: Initial integration with vizier Signed-off-by: Eryk Szpotanski --- tools/AutoTuner/pyproject.toml | 42 ++ tools/AutoTuner/requirements_vizier.txt | 3 + tools/AutoTuner/src/autotuner/distributed.py | 491 +-------------- .../AutoTuner/src/autotuner/parse_results.py | 114 ++++ tools/AutoTuner/src/autotuner/utils.py | 566 ++++++++++++++++++ tools/AutoTuner/src/autotuner/vizier.py | 466 ++++++++++++++ 6 files changed, 1221 insertions(+), 461 deletions(-) create mode 100644 tools/AutoTuner/pyproject.toml create mode 100644 tools/AutoTuner/requirements_vizier.txt create mode 100644 tools/AutoTuner/src/autotuner/parse_results.py create mode 100644 tools/AutoTuner/src/autotuner/utils.py create mode 100644 tools/AutoTuner/src/autotuner/vizier.py diff --git a/tools/AutoTuner/pyproject.toml b/tools/AutoTuner/pyproject.toml new file mode 100644 index 0000000000..20ffe48d1c --- /dev/null +++ b/tools/AutoTuner/pyproject.toml @@ -0,0 +1,42 @@ +[project] +name = "autotuner" +version = "0.0.1" +description = "This project provides a set of tools for tuning OpenROAD-flow-scripts parameter without user interference." +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD 3-Clause", +] +readme = "RAY_README.md" +dependencies = [] + +[project.optional-dependencies] +ray = [ + "ray[default,tune]==2.9.3", + "ax-platform>=0.3.3,<=0.3.7", + "hyperopt==0.2.7", + "nevergrad==1.0.2", + "optuna==3.6.0", + "pandas>=2.0,<=2.2.1", + "bayesian-optimization==1.4.0", + "colorama==0.4.6", + "tensorboard>=2.14.0,<=2.16.2", + "protobuf==3.20.3", + "SQLAlchemy==1.4.17", + "urllib3<=1.26.15", +] +vizier = [ + "google-vizier[jax]", +] + +[build-system] +requires = ["setuptools", "setuptools_scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src/"] +include = [ + "autotuner*", +] + +[tool.setuptools] +include-package-data = true diff --git a/tools/AutoTuner/requirements_vizier.txt b/tools/AutoTuner/requirements_vizier.txt new file mode 100644 index 0000000000..7a65c968f4 --- /dev/null +++ b/tools/AutoTuner/requirements_vizier.txt @@ -0,0 +1,3 @@ +google-vizier[jax] +toml +pandas diff --git a/tools/AutoTuner/src/autotuner/distributed.py b/tools/AutoTuner/src/autotuner/distributed.py index 93bea304d1..0c6246da25 100644 --- a/tools/AutoTuner/src/autotuner/distributed.py +++ b/tools/AutoTuner/src/autotuner/distributed.py @@ -3,23 +3,23 @@ Dependencies are documented in pip format at distributed-requirements.txt For both sweep and tune modes: - python3 distributed.py -h + python3 -m distributed -h Note: the order of the parameters matter. Arguments --design, --platform and --config are always required and should precede the . AutoTuner: - python3 distributed.py tune -h - python3 distributed.py --design gcd --platform sky130hd \ + python3 -m distributed tune -h + python3 -m distributed --design gcd --platform sky130hd \ --config ../designs/sky130hd/gcd/autotuner.json \ tune Example: Parameter sweeping: - python3 distributed.py sweep -h + python3 -m distributed sweep -h Example: - python3 distributed.py --design gcd --platform sky130hd \ + python3 -m distributed --design gcd --platform sky130hd \ --config distributed-sweep-example.json \ sweep """ @@ -27,13 +27,9 @@ import argparse import json import os -import re import sys -import glob -import subprocess from datetime import datetime from multiprocessing import cpu_count -from subprocess import run from itertools import product from uuid import uuid4 as uuid @@ -55,6 +51,15 @@ # import nevergrad as ng from ax.service.ax_client import AxClient +from utils import ( + add_common_args, + openroad, + parse_config, + read_config, + read_metrics, + run_command, +) + DATE = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") ORFS_URL = "https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts" FASTROUTE_TCL = "fastroute.tcl" @@ -80,7 +85,7 @@ def setup(self, config): # ////-DATE// repo_dir = os.getcwd() + "/../" * 6 self.repo_dir = os.path.abspath(repo_dir) - self.parameters = parse_config(config, path=os.getcwd()) + self.parameters = parse_config(config, args.platform, SDC_ORIGINAL, CONSTRAINTS_SDC, FR_ORIGINAL, FASTROUTE_TCL, path=os.getcwd()) self.step_ = 0 self.variant = f"variant-{self.__class__.__name__}-{self.trial_id}-or" @@ -88,9 +93,9 @@ def step(self): """ Run step experiment and compute its score. """ - metrics_file = openroad(self.repo_dir, self.parameters, self.variant) + metrics_file = openroad(args, self.repo_dir, self.parameters, self.variant, install_path=INSTALL_PATH, stage=args.to_stage) self.step_ += 1 - score = self.evaluate(self.read_metrics(metrics_file)) + score = self.evaluate(read_metrics(metrics_file, args.to_stage)) # Feed the score back to Tune. # return must match 'metric' used in tune.run() return {METRIC: score} @@ -103,6 +108,8 @@ def evaluate(self, metrics): """ error = "ERR" in metrics.values() not_found = "N/A" in metrics.values() + print("evaluate METRICS") + print(metrics) if error or not_found: return ERROR_METRIC gamma = (metrics["clk_period"] - metrics["worst_slack"]) / 10 @@ -110,46 +117,6 @@ def evaluate(self, metrics): score = score * (self.step_ / 100) ** (-1) + gamma * metrics["num_drc"] return score - @classmethod - def read_metrics(cls, file_name): - """ - Collects metrics to evaluate the user-defined objective function. - """ - with open(file_name) as file: - data = json.load(file) - clk_period = 9999999 - worst_slack = "ERR" - wirelength = "ERR" - num_drc = "ERR" - total_power = "ERR" - core_util = "ERR" - final_util = "ERR" - for stage, value in data.items(): - if stage == "constraints" and len(value["clocks__details"]) > 0: - clk_period = float(value["clocks__details"][0].split()[1]) - if stage == "floorplan" and "design__instance__utilization" in value: - core_util = value["design__instance__utilization"] - if stage == "detailedroute" and "route__drc_errors" in value: - num_drc = value["route__drc_errors"] - if stage == "detailedroute" and "route__wirelength" in value: - wirelength = value["route__wirelength"] - if stage == "detailedroute" and "timing__setup__ws" in value: - worst_slack = value["timing__setup__ws"] - if stage == "detailedroute" and "power__total" in value: - total_power = value["power__total"] - if stage == "detailedroute" and "design__instance__utilization" in value: - final_util = value["design__instance__utilization"] - ret = { - "clk_period": clk_period, - "worst_slack": worst_slack, - "wirelength": wirelength, - "num_drc": num_drc, - "total_power": total_power, - "core_util": core_util, - "final_util": final_util, - } - return ret - class PPAImprov(AutoTunerBase): """ @@ -188,6 +155,8 @@ def percent(x_1, x_2): def evaluate(self, metrics): error = "ERR" in metrics.values() or "ERR" in reference.values() not_found = "N/A" in metrics.values() or "N/A" in reference.values() + print("evaluate METRICS") + print(metrics) if error or not_found: return ERROR_METRIC ppa = self.get_ppa(metrics) @@ -196,367 +165,11 @@ def evaluate(self, metrics): return score -def read_config(file_name): - """ - Please consider inclusive, exclusive - Most type uses [min, max) - But, Quantization makes the upper bound inclusive. - e.g., qrandint and qlograndint uses [min, max] - step value is used for quantized type (e.g., quniform). Otherwise, write 0. - When min==max, it means the constant value - """ - - def read(path): - with open(os.path.abspath(path), "r") as file: - ret = file.read() - return ret - - def read_sweep(this): - return [*this["minmax"], this["step"]] - - def apply_condition(config, data): - # TODO: tune.sample_from only supports random search algorithm. - # To make conditional parameter for the other algorithms, different - # algorithms should take different methods (will be added) - if args.algorithm != "random": - return config - dp_pad_min = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["minmax"][0] - # dp_pad_max = data['CELL_PAD_IN_SITES_DETAIL_PLACEMENT']['minmax'][1] - dp_pad_step = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["step"] - if dp_pad_step == 1: - config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from( - lambda spec: tune.randint( - dp_pad_min, spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1 - ) - ) - if dp_pad_step > 1: - config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from( - lambda spec: tune.choice( - np.ndarray.tolist( - np.arange( - dp_pad_min, - spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1, - dp_pad_step, - ) - ) - ) - ) - return config - - def read_tune(this): - min_, max_ = this["minmax"] - if min_ == max_: - # Returning a choice of a single element allow pbt algorithm to - # work. pbt does not accept single values as tunable. - return tune.choice([min_]) - if this["type"] == "int": - if min_ == 0 and args.algorithm == "nevergrad": - print( - "[WARNING TUN-0011] NevergradSearch may not work " - "with lower bound value 0." - ) - if this["step"] == 1: - return tune.randint(min_, max_) - return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"]))) - if this["type"] == "float": - if this["step"] == 0: - return tune.uniform(min_, max_) - return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"]))) - return None - - def read_tune_ax(name, this): - dict_ = dict(name=name) - min_, max_ = this["minmax"] - if min_ == max_: - dict_["type"] = "fixed" - dict_["value"] = min_ - elif this["type"] == "int": - if this["step"] == 1: - dict_["type"] = "range" - dict_["bounds"] = [min_, max_] - dict_["value_type"] = "int" - else: - dict_["type"] = "choice" - dict_["values"] = tune.randint(min_, max_, this["step"]) - dict_["value_type"] = "int" - elif this["type"] == "float": - if this["step"] == 1: - dict_["type"] = "choice" - dict_["values"] = tune.choice( - np.ndarray.tolist(np.arange(min_, max_, this["step"])) - ) - dict_["value_type"] = "float" - else: - dict_["type"] = "range" - dict_["bounds"] = [min_, max_] - dict_["value_type"] = "float" - return dict_ - - # Check file exists and whether it is a valid JSON file. - assert os.path.isfile(file_name), f"File {file_name} not found." - try: - with open(file_name) as file: - data = json.load(file) - except json.JSONDecodeError: - raise ValueError(f"Invalid JSON file: {file_name}") - sdc_file = "" - fr_file = "" - if args.mode == "tune" and args.algorithm == "ax": - config = list() - else: - config = dict() - for key, value in data.items(): - if key == "best_result": - continue - if key == "_SDC_FILE_PATH" and value != "": - if sdc_file != "": - print("[WARNING TUN-0004] Overwriting SDC base file.") - sdc_file = read(f"{os.path.dirname(file_name)}/{value}") - continue - if key == "_FR_FILE_PATH" and value != "": - if fr_file != "": - print("[WARNING TUN-0005] Overwriting FastRoute base file.") - fr_file = read(f"{os.path.dirname(file_name)}/{value}") - continue - if not isinstance(value, dict): - config[key] = value - elif args.mode == "sweep": - config[key] = read_sweep(value) - elif args.mode == "tune" and args.algorithm != "ax": - config[key] = read_tune(value) - elif args.mode == "tune" and args.algorithm == "ax": - config.append(read_tune_ax(key, value)) - if args.mode == "tune": - config = apply_condition(config, data) - return config, sdc_file, fr_file - - -def parse_flow_variables(): - """ - Parse the flow variables from source - - Code: Makefile `vars` target output - - TODO: Tests. - - Output: - - flow_variables: set of flow variables - """ - cur_path = os.path.dirname(os.path.realpath(__file__)) - - # first, generate vars.tcl - makefile_path = os.path.join(cur_path, "../../../../flow/") - initial_path = os.path.abspath(os.getcwd()) - os.chdir(makefile_path) - result = subprocess.run(["make", "vars", f"PLATFORM={args.platform}"]) - if result.returncode != 0: - print(f"[ERROR TUN-0018] Makefile failed with error code {result.returncode}.") - sys.exit(1) - if not os.path.exists("vars.tcl"): - print(f"[ERROR TUN-0019] Makefile did not generate vars.tcl.") - sys.exit(1) - os.chdir(initial_path) - - # for code parsing, you need to parse from both scripts and vars.tcl file. - pattern = r"(?:::)?env\((.*?)\)" - files = glob.glob(os.path.join(cur_path, "../../../../flow/scripts/*.tcl")) - files.append(os.path.join(cur_path, "../../../../flow/vars.tcl")) - variables = set() - for file in files: - with open(file) as fp: - matches = re.findall(pattern, fp.read()) - for match in matches: - for variable in match.split("\n"): - variables.add(variable.strip().upper()) - return variables - - -def parse_config(config, path=os.getcwd()): - """ - Parse configuration received from tune into make variables. - """ - options = "" - sdc = {} - fast_route = {} - flow_variables = parse_flow_variables() - for key, value in config.items(): - # Keys that begin with underscore need special handling. - if key.startswith("_"): - # Variables to be injected into fastroute.tcl - if key.startswith("_FR_"): - fast_route[key.replace("_FR_", "", 1)] = value - # Variables to be injected into constraints.sdc - elif key.startswith("_SDC_"): - sdc[key.replace("_SDC_", "", 1)] = value - # Special substitution cases - elif key == "_PINS_DISTANCE": - options += f' PLACE_PINS_ARGS="-min_distance {value}"' - elif key == "_SYNTH_FLATTEN": - print( - "[WARNING TUN-0013] Non-flatten the designs are not " - "fully supported, ignoring _SYNTH_FLATTEN parameter." - ) - # Default case is VAR=VALUE - else: - # Sanity check: ignore all flow variables that are not tunable - if key not in flow_variables: - print(f"[ERROR TUN-0017] Variable {key} is not tunable.") - sys.exit(1) - options += f" {key}={value}" - if bool(sdc): - write_sdc(sdc, path) - options += f" SDC_FILE={path}/{CONSTRAINTS_SDC}" - if bool(fast_route): - write_fast_route(fast_route, path) - options += f" FASTROUTE_TCL={path}/{FASTROUTE_TCL}" - return options - - -def write_sdc(variables, path): - """ - Create a SDC file with parameters for current tuning iteration. - """ - # TODO: handle case where the reference file does not exist - new_file = SDC_ORIGINAL - for key, value in variables.items(): - if key == "CLK_PERIOD": - if new_file.find("set clk_period") != -1: - new_file = re.sub( - r"set clk_period .*\n(.*)", f"set clk_period {value}\n\\1", new_file - ) - else: - new_file = re.sub( - r"-period [0-9\.]+ (.*)", f"-period {value} \\1", new_file - ) - new_file = re.sub(r"-waveform [{}\s0-9\.]+[\s|\n]", "", new_file) - elif key == "UNCERTAINTY": - if new_file.find("set uncertainty") != -1: - new_file = re.sub( - r"set uncertainty .*\n(.*)", - f"set uncertainty {value}\n\\1", - new_file, - ) - else: - new_file += f"\nset uncertainty {value}\n" - elif key == "IO_DELAY": - if new_file.find("set io_delay") != -1: - new_file = re.sub( - r"set io_delay .*\n(.*)", f"set io_delay {value}\n\\1", new_file - ) - else: - new_file += f"\nset io_delay {value}\n" - file_name = path + f"/{CONSTRAINTS_SDC}" - with open(file_name, "w") as file: - file.write(new_file) - return file_name - - -def write_fast_route(variables, path): - """ - Create a FastRoute Tcl file with parameters for current tuning iteration. - """ - # TODO: handle case where the reference file does not exist - layer_cmd = "set_global_routing_layer_adjustment" - new_file = FR_ORIGINAL - for key, value in variables.items(): - if key.startswith("LAYER_ADJUST"): - layer = key.lstrip("LAYER_ADJUST") - # If there is no suffix (i.e., layer name) apply adjust to all - # layers. - if layer == "": - new_file += "\nset_global_routing_layer_adjustment" - new_file += " $::env(MIN_ROUTING_LAYER)" - new_file += "-$::env(MAX_ROUTING_LAYER)" - new_file += f" {value}" - elif re.search(f"{layer_cmd}.*{layer}", new_file): - new_file = re.sub( - f"({layer_cmd}.*{layer}).*\n(.*)", f"\\1 {value}\n\\2", new_file - ) - else: - new_file += f"\n{layer_cmd} {layer} {value}\n" - elif key == "GR_SEED": - new_file += f"\nset_global_routing_random -seed {value}\n" - file_name = path + f"/{FASTROUTE_TCL}" - with open(file_name, "w") as file: - file.write(new_file) - return file_name - - -def run_command(cmd, timeout=None, stderr_file=None, stdout_file=None, fail_fast=False): - """ - Wrapper for subprocess.run - Allows to run shell command, control print and exceptions. - """ - process = run( - cmd, timeout=timeout, capture_output=True, text=True, check=False, shell=True - ) - if stderr_file is not None and process.stderr != "": - with open(stderr_file, "a") as file: - file.write(f"\n\n{cmd}\n{process.stderr}") - if stdout_file is not None and process.stdout != "": - with open(stdout_file, "a") as file: - file.write(f"\n\n{cmd}\n{process.stdout}") - if args.verbose >= 1: - print(process.stderr) - if args.verbose >= 2: - print(process.stdout) - - if fail_fast and process.returncode != 0: - raise RuntimeError - - @ray.remote def openroad_distributed(repo_dir, config, path): """Simple wrapper to run openroad distributed with Ray.""" - config = parse_config(config) - openroad(repo_dir, config, str(uuid()), path=path) - - -def openroad(base_dir, parameters, flow_variant, path=""): - """ - Run OpenROAD-flow-scripts with a given set of parameters. - """ - # Make sure path ends in a slash, i.e., is a folder - flow_variant = f"{args.experiment}/{flow_variant}" - if path != "": - log_path = f"{path}/{flow_variant}/" - report_path = log_path.replace("logs", "reports") - run_command(f"mkdir -p {log_path}") - run_command(f"mkdir -p {report_path}") - else: - log_path = report_path = os.getcwd() + "/" - - export_command = f"export PATH={INSTALL_PATH}/OpenROAD/bin" - export_command += f":{INSTALL_PATH}/yosys/bin:$PATH" - export_command += " && " - - make_command = export_command - make_command += f"make -C {base_dir}/flow DESIGN_CONFIG=designs/" - make_command += f"{args.platform}/{args.design}/config.mk route" - make_command += f" PLATFORM={args.platform}" - make_command += f" FLOW_VARIANT={flow_variant} {parameters}" - make_command += f" EQUIVALENCE_CHECK=0" - make_command += f" NPROC={args.openroad_threads} SHELL=bash" - run_command( - make_command, - timeout=args.timeout, - stderr_file=f"{log_path}error-make-finish.log", - stdout_file=f"{log_path}make-finish-stdout.log", - ) - - metrics_file = os.path.join(report_path, "metrics.json") - metrics_command = export_command - metrics_command += f"{base_dir}/flow/util/genMetrics.py -x" - metrics_command += f" -v {flow_variant}" - metrics_command += f" -d {args.design}" - metrics_command += f" -p {args.platform}" - metrics_command += f" -o {metrics_file}" - run_command( - metrics_command, - stderr_file=f"{log_path}error-metrics.log", - stdout_file=f"{log_path}metrics-stdout.log", - ) - - return metrics_file + config = parse_config(args, config, SDC_ORIGINAL, CONSTRAINTS_SDC, FR_ORIGINAL, FASTROUTE_TCL) + openroad(args, repo_dir, config, str(uuid()), path=path, install_path=INSTALL_PATH, stage=args.to_stage) def clone(path): @@ -564,13 +177,13 @@ def clone(path): Clone base repo in the remote machine. Only used for Kubernetes at GCP. """ if args.git_clone: - run_command(f"rm -rf {path}") + run_command(args, f"rm -rf {path}") if not os.path.isdir(f"{path}/.git"): git_command = "git clone --depth 1 --recursive --single-branch" git_command += f" {args.git_clone_args}" git_command += f" --branch {args.git_orfs_branch}" git_command += f" {args.git_url} {path}" - run_command(git_command) + run_command(args, git_command) def build(base, install): @@ -592,7 +205,7 @@ def build(base, install): if args.git_latest: build_command += " --latest" build_command += f' {args.build_args}"' - run_command(build_command) + run_command(args, build_command) @ray.remote @@ -620,30 +233,9 @@ def parse_arguments(): tune_parser = subparsers.add_parser("tune") _ = subparsers.add_parser("sweep") - # DUT - parser.add_argument( - "--design", - type=str, - metavar="", - required=True, - help="Name of the design for Autotuning.", - ) - parser.add_argument( - "--platform", - type=str, - metavar="", - required=True, - help="Name of the platform for Autotuning.", - ) + add_common_args(parser) # Experiment Setup - parser.add_argument( - "--config", - type=str, - metavar="", - required=True, - help="Configuration file that sets which knobs to use for Autotuning.", - ) parser.add_argument( "--experiment", type=str, @@ -652,13 +244,6 @@ def parse_arguments(): help="Experiment name. This parameter is used to prefix the" " FLOW_VARIANT and to set the Ray log destination.", ) - parser.add_argument( - "--timeout", - type=float, - metavar="", - default=None, - help="Time limit (in hours) for each trial run. Default is no limit.", - ) tune_parser.add_argument( "--resume", action="store_true", help="Resume previous run." ) @@ -779,13 +364,6 @@ def parse_arguments(): default=int(np.floor(cpu_count() / 2)), help="Max number of concurrent jobs.", ) - parser.add_argument( - "--openroad_threads", - type=int, - metavar="", - default=16, - help="Max number of threads openroad can use.", - ) parser.add_argument( "--server", type=str, @@ -801,15 +379,6 @@ def parse_arguments(): help="The port of Ray server to connect.", ) - parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="Verbosity level.\n\t0: only print Ray status\n\t1: also print" - " training stderr\n\t2: also print training stdout.", - ) - arguments = parser.parse_args() if arguments.mode == "tune": arguments.algorithm = arguments.algorithm.lower() @@ -947,12 +516,12 @@ def sweep(): print("[INFO TUN-0010] Sweep complete.") -if __name__ == "__main__": +if __name__=="__main__": args = parse_arguments() # Read config and original files before handling where to run in case we # need to upload the files. - config_dict, SDC_ORIGINAL, FR_ORIGINAL = read_config(os.path.abspath(args.config)) + config_dict, SDC_ORIGINAL, FR_ORIGINAL = read_config(os.path.abspath(args.config), args.mode, args.algorithm) # Connect to remote Ray server if any, otherwise will run locally if args.server is not None: @@ -990,7 +559,7 @@ def sweep(): TrainClass = set_training_class(args.eval) # PPAImprov requires a reference file to compute training scores. if args.eval == "ppa-improv": - reference = PPAImprov.read_metrics(args.reference) + reference = read_metrics(args.reference, args.to_stage) tune_args = dict( name=args.experiment, diff --git a/tools/AutoTuner/src/autotuner/parse_results.py b/tools/AutoTuner/src/autotuner/parse_results.py new file mode 100644 index 0000000000..b9023341c0 --- /dev/null +++ b/tools/AutoTuner/src/autotuner/parse_results.py @@ -0,0 +1,114 @@ +import argparse +import pandas as pd +import json +import re +import sys +from pathlib import Path + +METRICS = ( + "clk_period-worst_slack", + # "worst_slack", + "total_power", + "core_util", + "final_util", + "design_area", + "core_area", + "die_area", + "last_successful_stage", +) + + +def normalize_level_number(name: str) -> str: + """ + Adds zero padding for numbers in name. + + Parameters + ---------- + name : str + Name of the design + + Returns + ------- + str + Name with padded numbers + """ + + def _replace(match: re.Match) -> str: + number = int(match.group(1)) + return f"_{number:05}_" + + return re.sub(r"_([0-9]+)_", _replace, name) + + +def read_config(conf): + params = [] + for key, value in conf.items(): + if value["type"] != "fixed": + params.append(key) + columns = ("Level", *params, *METRICS) + return params, columns + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-r", + "--results", + help="Directory containing optimization results (JSON files)", + type=Path, + required=True, + ) + parser.add_argument( + "--to-markdown", + help="Path where markdown file with results will be saved", + default=None, + type=Path, + ) + parser.add_argument( + "--to-html", + help="Path where html file with results will be saved", + default=None, + type=Path, + ) + + args = parser.parse_args(sys.argv[1:]) + + rows = [] + params, columns = None, None + for result in sorted( + args.results.glob("*.json"), + key=lambda x: normalize_level_number(x.with_suffix("").name), + ): + design = result.with_suffix("").name + with result.open("r") as fd: + result_json = json.load(fd) + if params is None or columns is None: + params, columns = read_config(result_json["config"]) + optimals = result_json["optimals"] + not_finished = False + if 11 in set(opt["evaluation"]["last_successful_stage"] for opt in optimals): + opts = sorted( + filter( + lambda x: x["evaluation"]["last_successful_stage"] == 11, optimals + ), + key=lambda x: x["evaluation"]["core_util"], + reverse=True, + ) + else: + opts = [optimals[0]] + for opt in opts: + if opt["evaluation"]["last_successful_stage"] < 11: + if not_finished: + continue + not_finished = True + rows.append( + [design] + + [opt["params"][p] for p in params] + + [opt["evaluation"][e] for e in METRICS] + ) + rows.append(["" for _ in columns]) + result_df = pd.DataFrame(rows[:-1], columns=columns) + if args.to_markdown: + result_df.to_markdown(args.to_markdown, index=False) + if args.to_html: + result_df.to_html(args.to_html, index=False) diff --git a/tools/AutoTuner/src/autotuner/utils.py b/tools/AutoTuner/src/autotuner/utils.py new file mode 100644 index 0000000000..d9f80b193a --- /dev/null +++ b/tools/AutoTuner/src/autotuner/utils.py @@ -0,0 +1,566 @@ +import argparse +import glob +import json +import os +import re +import sys +from subprocess import run + +import numpy as np + +SDC_TEMPLATE = """ +set clk_name core_clock +set clk_port_name clk +set clk_period 2000 +set clk_io_pct 0.2 + +set clk_port [get_ports $clk_port_name] + +create_clock -name $clk_name -period $clk_period $clk_port + +set non_clock_inputs [lsearch -inline -all -not -exact [all_inputs] $clk_port] + +set_input_delay [expr $clk_period * $clk_io_pct] -clock $clk_name $non_clock_inputs +set_output_delay [expr $clk_period * $clk_io_pct] -clock $clk_name [all_outputs] +""" +STAGE_TO_METRICS = { + "route": "detailedroute", + "place": "detailedplace", + "final": "finish", +} + +def write_sdc(variables, path, sdc_original, constraints_sdc): + """ + Create a SDC file with parameters for current tuning iteration. + """ + # TODO: handle case where the reference file does not exist + new_file = sdc_original + for key, value in variables.items(): + if key == "CLK_PERIOD": + if new_file.find("set clk_period") != -1: + new_file = re.sub( + r"set clk_period .*\n(.*)", f"set clk_period {value}\n\\1", new_file + ) + else: + new_file = re.sub( + r"-period [0-9\.]+ (.*)", f"-period {value} \\1", new_file + ) + new_file = re.sub(r"-waveform [{}\s0-9\.]+[\s|\n]", "", new_file) + elif key == "UNCERTAINTY": + if new_file.find("set uncertainty") != -1: + new_file = re.sub( + r"set uncertainty .*\n(.*)", + f"set uncertainty {value}\n\\1", + new_file, + ) + else: + new_file += f"\nset uncertainty {value}\n" + elif key == "IO_DELAY": + if new_file.find("set io_delay") != -1: + new_file = re.sub( + r"set io_delay .*\n(.*)", f"set io_delay {value}\n\\1", new_file + ) + else: + new_file += f"\nset io_delay {value}\n" + file_name = path + f"/{constraints_sdc}" + with open(file_name, "w") as file: + file.write(new_file) + return file_name + + +def write_fast_route(variables, path, fr_original, fastroute_tcl): + """ + Create a FastRoute Tcl file with parameters for current tuning iteration. + """ + # TODO: handle case where the reference file does not exist + layer_cmd = "set_global_routing_layer_adjustment" + new_file = fr_original + for key, value in variables.items(): + if key.startswith("LAYER_ADJUST"): + layer = key.lstrip("LAYER_ADJUST") + # If there is no suffix (i.e., layer name) apply adjust to all + # layers. + if layer == "": + new_file += "\nset_global_routing_layer_adjustment" + new_file += " $::env(MIN_ROUTING_LAYER)" + new_file += "-$::env(MAX_ROUTING_LAYER)" + new_file += f" {value}" + elif re.search(f"{layer_cmd}.*{layer}", new_file): + new_file = re.sub( + f"({layer_cmd}.*{layer}).*\n(.*)", f"\\1 {value}\n\\2", new_file + ) + else: + new_file += f"\n{layer_cmd} {layer} {value}\n" + elif key == "GR_SEED": + new_file += f"\nset_global_routing_random -seed {value}\n" + file_name = path + f"/{fastroute_tcl}" + with open(file_name, "w") as file: + file.write(new_file) + return file_name + + +def parse_flow_variables(platform): + """ + Parse the flow variables from source + - Code: Makefile `vars` target output + + TODO: Tests. + + Output: + - flow_variables: set of flow variables + """ + cur_path = os.path.dirname(os.path.realpath(__file__)) + + # first, generate vars.tcl + makefile_path = os.path.join(cur_path, "../../../../flow/") + initial_path = os.path.abspath(os.getcwd()) + os.chdir(makefile_path) + result = run(["make", "vars", f"PLATFORM={platform}"]) + if result.returncode != 0: + print(f"[ERROR TUN-0018] Makefile failed with error code {result.returncode}.") + sys.exit(1) + if not os.path.exists("vars.tcl"): + print("[ERROR TUN-0019] Makefile did not generate vars.tcl.") + sys.exit(1) + os.chdir(initial_path) + + # for code parsing, you need to parse from both scripts and vars.tcl file. + pattern = r"(?:::)?env\((.*?)\)" + files = glob.glob(os.path.join(cur_path, "../../../../flow/scripts/*.tcl")) + files.append(os.path.join(cur_path, "../../../../flow/vars.tcl")) + variables = set() + for file in files: + with open(file) as fp: + matches = re.findall(pattern, fp.read()) + for match in matches: + for variable in match.split("\n"): + variables.add(variable.strip().upper()) + return variables + + +def parse_config( + config, + platform, + sdc_original, + constraints_sdc, + fr_original, + fastroute_tcl, + path=os.getcwd(), +): + """ + Parse configuration received from tune into make variables. + """ + options = "" + sdc = {} + fast_route = {} + flow_variables = parse_flow_variables(platform) + for key, value in config.items(): + # Keys that begin with underscore need special handling. + if key.startswith("_"): + # Variables to be injected into fastroute.tcl + if key.startswith("_FR_"): + fast_route[key.replace("_FR_", "", 1)] = value + # Variables to be injected into constraints.sdc + elif key.startswith("_SDC_"): + sdc[key.replace("_SDC_", "", 1)] = value + # Special substitution cases + elif key == "_PINS_DISTANCE": + options += f' PLACE_PINS_ARGS="-min_distance {value}"' + elif key == "_SYNTH_FLATTEN": + print( + "[WARNING TUN-0013] Non-flatten the designs are not " + "fully supported, ignoring _SYNTH_FLATTEN parameter." + ) + # Default case is VAR=VALUE + else: + # Sanity check: ignore all flow variables that are not tunable + if key not in flow_variables: + print(f"[ERROR TUN-0017] Variable {key} is not tunable.") + sys.exit(1) + options += f" {key}={value}" + if bool(sdc): + write_sdc(sdc, path, sdc_original, constraints_sdc) + options += f" SDC_FILE={path}/{constraints_sdc}" + if bool(fast_route): + write_fast_route(fast_route, path, fr_original, fastroute_tcl) + options += f" FASTROUTE_TCL={path}/{fastroute_tcl}" + return options + + +def run_command( + args, cmd, timeout=None, stderr_file=None, stdout_file=None, fail_fast=False +): + """ + Wrapper for subprocess.run + Allows to run shell command, control print and exceptions. + """ + process = run( + cmd, timeout=timeout, capture_output=True, text=True, check=False, shell=True + ) + if stderr_file is not None and process.stderr != "": + with open(stderr_file, "a") as file: + file.write(f"\n\n{cmd}\n{process.stderr}") + if stdout_file is not None and process.stdout != "": + with open(stdout_file, "a") as file: + file.write(f"\n\n{cmd}\n{process.stdout}") + if args.verbose >= 1: + print(process.stderr) + if args.verbose >= 2: + print(process.stdout) + + if fail_fast and process.returncode != 0: + raise RuntimeError + + +def openroad( + args, + base_dir, + parameters, + flow_variant, + path="", + install_path=os.path.abspath("../tools/install"), + stage="", +): + """ + Run OpenROAD-flow-scripts with a given set of parameters. + """ + # Make sure path ends in a slash, i.e., is a folder + flow_variant = f"{args.experiment}/{flow_variant}" + if path != "": + log_path = f"{path}/{flow_variant}/" + report_path = log_path.replace("logs", "reports") + run_command(args, f"mkdir -p {log_path}") + run_command(args, f"mkdir -p {report_path}") + else: + log_path = report_path = os.getcwd() + "/" + + export_command = f"export PATH={install_path}/OpenROAD/bin" + export_command += f":{install_path}/yosys/bin:$PATH" + export_command += " && " + + make_command = export_command + make_command += f"make -C {base_dir}/flow DESIGN_CONFIG=designs/" + make_command += f"{args.platform}/{args.design}/config.mk {stage}" + make_command += f" PLATFORM={args.platform}" + make_command += f" FLOW_VARIANT={flow_variant} {parameters}" + make_command += " EQUIVALENCE_CHECK=0" + make_command += f" NPROC={args.openroad_threads} SHELL=bash" + run_command( + args, + make_command, + timeout=args.timeout, + stderr_file=f"{log_path}error-make-finish.log", + stdout_file=f"{log_path}make-finish-stdout.log", + ) + + metrics_file = os.path.join(report_path, "metrics.json") + metrics_command = export_command + metrics_command += f"{base_dir}/flow/util/genMetrics.py -x" + metrics_command += f" -v {flow_variant}" + metrics_command += f" -d {args.design}" + metrics_command += f" -p {args.platform}" + metrics_command += f" -o {metrics_file}" + run_command( + args, + metrics_command, + stderr_file=f"{log_path}error-metrics.log", + stdout_file=f"{log_path}metrics-stdout.log", + ) + + return metrics_file + + +STAGES = list( + enumerate( + [ + "synth", + "floorplan", + "floorplan_io", + "floorplan_tdms", + "floorplan_macro", + "floorplan_tap", + "floorplan_pdn", + "globalplace", + "detailedplace", + "cts", + "globalroute", + "detailedroute", + ] + ) +) + + +def read_metrics(file_name, stage=""): + """ + Collects metrics to evaluate the user-defined objective function. + """ + metric_name = STAGE_TO_METRICS.get(stage if stage else "final", stage) + with open(file_name) as file: + data = json.load(file) + clk_period = 9999999 + worst_slack = "ERR" + wirelength = "ERR" + num_drc = "ERR" + total_power = "ERR" + core_util = "ERR" + final_util = "ERR" + design_area = "ERR" + die_area = "ERR" + core_area = "ERR" + last_stage = -1 + for stage_name, value in data.items(): + if stage_name == "constraints" and len(value["clocks__details"]) > 0: + clk_period = float(value["clocks__details"][0].split()[1]) + if stage_name == "floorplan" and "design__instance__utilization" in value: + core_util = value["design__instance__utilization"] + if stage_name == "detailedroute" and "route__drc_errors" in value: + num_drc = value["route__drc_errors"] + if stage_name == "detailedroute" and "route__wirelength" in value: + wirelength = value["route__wirelength"] + if stage_name == metric_name and "timing__setup__ws" in value: + worst_slack = value["timing__setup__ws"] + if stage_name == metric_name and "power__total" in value: + total_power = value["power__total"] + if stage_name == metric_name and "design__instance__utilization" in value: + final_util = value["design__instance__utilization"] + if stage_name == metric_name and "design__instance__area" in value: + design_area = value["design__instance__area"] + if stage_name == metric_name and "design__core__area" in value: + core_area = value["design__core__area"] + if stage_name == metric_name and "design__die__area" in value: + die_area = value["design__die__area"] + for i, stage_name in reversed(STAGES): + if stage_name in data and [d for d in data[stage_name].values() if d != "ERR"]: + last_stage = i + break + ret = { + "clk_period": clk_period, + "worst_slack": worst_slack, + "total_power": total_power, + "core_util": core_util, + "final_util": final_util, + "design_area": design_area, + "core_area": core_area, + "die_area": die_area, + "last_successful_stage": last_stage, + } | ({ + "wirelength": wirelength, + "num_drc": num_drc, + } if metric_name in ("detailedroute", "finish") else {}) + return ret + + +def read_config(file_name, mode, algorithm): + """ + Please consider inclusive, exclusive + Most type uses [min, max) + But, Quantization makes the upper bound inclusive. + e.g., qrandint and qlograndint uses [min, max] + step value is used for quantized type (e.g., quniform). Otherwise, write 0. + When min==max, it means the constant value + """ + + def read(path): + with open(os.path.abspath(path), "r") as file: + ret = file.read() + return ret + + def read_sweep(this): + return [*this["minmax"], this["step"]] + + def apply_condition(config, data): + from ray import tune + + # TODO: tune.sample_from only supports random search algorithm. + # To make conditional parameter for the other algorithms, different + # algorithms should take different methods (will be added) + if algorithm != "random": + return config + dp_pad_min = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["minmax"][0] + # dp_pad_max = data['CELL_PAD_IN_SITES_DETAIL_PLACEMENT']['minmax'][1] + dp_pad_step = data["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"]["step"] + if dp_pad_step == 1: + config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from( + lambda spec: tune.randint( + dp_pad_min, spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1 + ) + ) + if dp_pad_step > 1: + config["CELL_PAD_IN_SITES_DETAIL_PLACEMENT"] = tune.sample_from( + lambda spec: tune.choice( + np.ndarray.tolist( + np.arange( + dp_pad_min, + spec.config.CELL_PAD_IN_SITES_GLOBAL_PLACEMENT + 1, + dp_pad_step, + ) + ) + ) + ) + return config + + def read_tune(this): + from ray import tune + + min_, max_ = this["minmax"] + if min_ == max_: + # Returning a choice of a single element allow pbt algorithm to + # work. pbt does not accept single values as tunable. + return tune.choice([min_]) + if this["type"] == "int": + if min_ == 0 and algorithm == "nevergrad": + print( + "[WARNING TUN-0011] NevergradSearch may not work " + "with lower bound value 0." + ) + if this["step"] == 1: + return tune.randint(min_, max_) + return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"]))) + if this["type"] == "float": + if this["step"] == 0: + return tune.uniform(min_, max_) + return tune.choice(np.ndarray.tolist(np.arange(min_, max_, this["step"]))) + return None + + def read_tune_ax(name, this): + from ray import tune + + dict_ = dict(name=name) + min_, max_ = this["minmax"] + if min_ == max_: + dict_["type"] = "fixed" + dict_["value"] = min_ + elif this["type"] == "int": + if this["step"] == 1: + dict_["type"] = "range" + dict_["bounds"] = [min_, max_] + dict_["value_type"] = "int" + else: + dict_["type"] = "choice" + dict_["values"] = tune.randint(min_, max_, this["step"]) + dict_["value_type"] = "int" + elif this["type"] == "float": + if this["step"] == 1: + dict_["type"] = "choice" + dict_["values"] = tune.choice( + np.ndarray.tolist(np.arange(min_, max_, this["step"])) + ) + dict_["value_type"] = "float" + else: + dict_["type"] = "range" + dict_["bounds"] = [min_, max_] + dict_["value_type"] = "float" + return dict_ + + def read_vizier(this): + dict_ = {} + min_, max_ = this["minmax"] + dict_["value"] = (min_, max_) + if "scale_type" in this: + dict_["scale_type"] = this["scale_type"] + if min_ == max_: + dict_["type"] = "fixed" + elif this["type"] == "int": + dict_["type"] = "int" + elif this["type"] == "float": + dict_["type"] = "float" + return dict_ + + # Check file exists and whether it is a valid JSON file. + assert os.path.isfile(file_name), f"File {file_name} not found." + try: + with open(file_name) as file: + data = json.load(file) + except json.JSONDecodeError: + raise ValueError(f"Invalid JSON file: {file_name}") + sdc_file = "" + fr_file = "" + if mode == "tune" and algorithm == "ax": + config = list() + else: + config = dict() + for key, value in data.items(): + if key == "best_result": + continue + if key == "_SDC_FILE_PATH" and value != "": + if sdc_file != "": + print("[WARNING TUN-0004] Overwriting SDC base file.") + try: + sdc_file = read(f"{os.path.dirname(file_name)}/{value}") + except FileNotFoundError: + sdc_file = SDC_TEMPLATE + continue + if key == "_FR_FILE_PATH" and value != "": + if fr_file != "": + print("[WARNING TUN-0005] Overwriting FastRoute base file.") + fr_file = read(f"{os.path.dirname(file_name)}/{value}") + continue + if not isinstance(value, dict): + config[key] = value + elif mode == "sweep": + config[key] = read_sweep(value) + elif mode == "tune" and algorithm != "ax": + config[key] = read_tune(value) + elif mode == "tune" and algorithm == "ax": + config.append(read_tune_ax(key, value)) + elif mode == "vizier": + config[key] = read_vizier(value) + if mode == "tune": + config = apply_condition(config, data) + return config, sdc_file, fr_file + + +def add_common_args(parser: argparse.ArgumentParser): + # DUT + parser.add_argument( + "--design", + type=str, + metavar="", + required=True, + help="Name of the design for Autotuning.", + ) + parser.add_argument( + "--platform", + type=str, + metavar="", + required=True, + help="Name of the platform for Autotuning.", + ) + # Experiment Setup + parser.add_argument( + "--config", + type=str, + metavar="", + required=True, + help="Configuration file that sets which knobs to use for Autotuning.", + ) + parser.add_argument( + "--to-stage", + type=str, + choices=("floorplan", "place", "cts", "route", "finish"), + default=None, + help="Run ORFS only to the given stage (inclusive)", + ) + parser.add_argument( + "--timeout", + type=float, + metavar="", + default=None, + help="Time limit (in hours) for each trial run. Default is no limit.", + ) + # Workload + parser.add_argument( + "--openroad_threads", + type=int, + metavar="", + default=16, + help="Max number of threads openroad can use.", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Verbosity level.\n\t0: only print status\n\t1: also print" + " training stderr\n\t2: also print training stdout.", + ) diff --git a/tools/AutoTuner/src/autotuner/vizier.py b/tools/AutoTuner/src/autotuner/vizier.py new file mode 100644 index 0000000000..3d2e3b0bb5 --- /dev/null +++ b/tools/AutoTuner/src/autotuner/vizier.py @@ -0,0 +1,466 @@ +import argparse +import json +import logging +import os +import sys +import time +import traceback +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime +from pathlib import Path +from tqdm import tqdm +from typing import Dict, Tuple + +from utils import add_common_args, openroad, parse_config, read_config, read_metrics +from vizier import service +from vizier.service import clients, servers +from vizier.service import pyvizier as vz + +ORFS = list(Path(__file__).absolute().parents)[4] +CONSTRAINTS_SDC = "constraint.sdc" +FASTROUTE_TCL = "fastroute.tcl" +METRIC_TO_GOAL = { + "worst_slack": vz.ObjectiveMetricGoal.MAXIMIZE, + "clk_period-worst_slack": vz.ObjectiveMetricGoal.MINIMIZE, + "total_power": vz.ObjectiveMetricGoal.MINIMIZE, + "core_util": vz.ObjectiveMetricGoal.MAXIMIZE, + "final_util": vz.ObjectiveMetricGoal.MAXIMIZE, + "design_area": vz.ObjectiveMetricGoal.MINIMIZE, + "core_area": vz.ObjectiveMetricGoal.MINIMIZE, + "die_area": vz.ObjectiveMetricGoal.MINIMIZE, + "last_successful_stage": vz.ObjectiveMetricGoal.MAXIMIZE, +} +GOAL_TO_VALUE = { + vz.ObjectiveMetricGoal.MINIMIZE: float("inf"), + vz.ObjectiveMetricGoal.MAXIMIZE: float("-inf"), +} +MAP_SCALE_TYPE = { + "linear": vz.ScaleType.LINEAR, + "log": vz.ScaleType.LOG, + "rlog": vz.ScaleType.REVERSE_LOG, +} +DATE = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") +LOG = logging.Logger(Path(__name__).with_suffix("").name) + + +def evaluate( + args: argparse.Namespace, + params: Dict, + iteration: int, + suggestion: int, + install_path: Path, +) -> Tuple[Dict[str, float], str, float]: + """ + Runs ORFS and calculates metrics. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + params : Dict + Parameters to evaluate + iteration : int + Current iteration + suggestion : int + Current suggestion number + install_path : Path + Path to the folder with installed ORFS binaries + + Returns + ------- + Tuple[Dict[str, float], str, float] + Dictionary with metrics, name of variant and duration of ORFS run + """ + variant = f"variant-{iteration}-{suggestion}" + try: + # Prepare ORFS options + options = parse_config( + params, + args.platform, + args.sdc_file, + CONSTRAINTS_SDC, + args.fr_file, + FASTROUTE_TCL, + ) + t = time.time() + # Run flow + metric_file = openroad( + args, + str(args.orfs), + options, + variant, + path=f"logs/{args.platform}/{args.design}", + install_path=str(install_path), + stage=args.to_stage, + ) + duration = time.time() - t + metrics = read_metrics(metric_file, stage=args.to_stage) + # Calculate difference of clock period and worst slack + if metrics["clk_period"] != 9999999 and metrics["worst_slack"] != "ERR": + metrics["clk_period-worst_slack"] = ( + metrics["clk_period"] - metrics["worst_slack"] + ) + else: + metrics["clk_period-worst_slack"] = "ERR" + + # Copy and normalize metrics + results = {} + for metric in args.use_metrics: + value = metrics[metric] + results[metric] = ( + float(value) + if value != "ERR" + else GOAL_TO_VALUE[METRIC_TO_GOAL[metric]] + ) + if results["last_successful_stage"] <= 6 and results["core_util"] < float( + "inf" + ): + # Invert core util, as for smaller values design should be easier to built + results["core_util"] *= -1 + return results, variant, duration + except Exception as ex: + LOG.error(f"Exception during {args.design} {variant}: {ex}", file=sys.stderr) + LOG.error("\n".join(traceback.format_tb(ex.__traceback__)), file=sys.stderr) + results = {} + for metric, goal in args.use_metrics: + results[metric] = GOAL_TO_VALUE[METRIC_TO_GOAL[metric]] + return results, variant, 0.0 + + +def parallel_evaluate(tup: Tuple[argparse.Namespace, Dict, int, int]) -> Dict: + """ + Wrapper for evaluate, run in thread pool. + + Parameters + ---------- + tup : Tuple[argparse.Namespace, Dict, int, int] + Tupled evaluate input + + Returns + ------- + Dict + Results of evaluation with additional data + """ + args, suggestion, i, s, install_path = tup + LOG.info(f"It {i} sug {s} params: {suggestion}") + objective, variant, duration = evaluate(args, suggestion, i, s, install_path) + LOG.info(f"It {i} sug {s} metric: {objective}") + return { + "iterations": i, + "suggestion": s, + "params": suggestion, + "evaluation": objective, + "variant": variant, + "duration": duration, + } + + +def register_param( + args: argparse.Namespace, problem: vz.ProblemStatement, name: str, conf: Dict +): + """ + Registers parameters in Vizier problem statement. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + problem : vz.ProblemStatement + Vizier problem statement + name : str + Name of the parameter + conf : Dict + Parameter config + """ + if conf["type"] == "fixed": + problem.search_space.root.add_discrete_param( + name, + feasible_values=[conf["value"][0]], + ) + else: + map_func = { + "float": problem.search_space.root.add_float_param, + "int": problem.search_space.root.add_int_param, + } + map_func[conf.get("type", "float")]( + name, + min_value=conf["value"][0], + max_value=conf["value"][1], + scale_type=MAP_SCALE_TYPE[conf.get("scale_type", "linear")], + ) + + +def main( + args: argparse.Namespace, + config: Dict, + install_path: Path, + server_endpoint: str = None, +) -> Dict: + """ + Converts config to Vizier problem definition and runs optimization. + + Parameters + ---------- + args : argparse.Namespace + Optimization arguments + config : Dict + Optimization configuration + install_path : Path + Path to the folder with installed ORFS binaries + server_endpoint : str + URL pointing to Vizier server + + Returns + ------- + Dict + Results of optimization, containing 'config', 'population' + and found 'optimals' + """ + results = {"config": config, "populations": [], "optimals": []} + + problem = vz.ProblemStatement() + for key, value in config.items(): + register_param(args, problem, key, value) + for metric, goal in METRIC_TO_GOAL.items(): + problem.metric_information.append(vz.MetricInformation(metric, goal=goal)) + + study_config = vz.StudyConfig.from_problem(problem) + study_config.algorithm = args.algorithm + + # Vizier Client setup + if server_endpoint: + clients.environment_variables.server_endpoint = server_endpoint + study_client = clients.Study.from_study_config( + study_config, owner="owner", study_id=f"{args.experiment}-{args.design}" + ) + + state = study_client.materialize_state() + start_iteration = 0 + # Check if experiment should be continued + if state == vz.StudyState.COMPLETED or state == vz.StudyState.ABORTED: + trials = list(study_client.trials().get()) + last_iteration = max( + map(lambda x: int(x.metadata.get("iteration", -1)), trials) + ) + start_iteration = last_iteration + 1 + if start_iteration <= args.iterations - 1: + LOG.warn(f"Trying to restart experiment (previously {state})") + study_client.set_state(vz.StudyState.ACTIVE) + + # Run iterations + for i, s in zip( + range(start_iteration, args.iterations), args.suggestions[start_iteration:] + ): + try: + suggestions = study_client.suggest(count=s) + with ThreadPoolExecutor(args.workers) as pool: + population = pool.map( + parallel_evaluate, + [ + (args, suggestion.parameters, i, s, install_path) + for s, suggestion in enumerate(suggestions) + ], + ) + tqdm_population = tqdm(population) + tqdm_population.set_description(f"Iteration {i}/{args.iterations}") + for p in tqdm_population: + results["populations"].append(p) + final_measurement = vz.Measurement(p["evaluation"]) + suggestions[p["suggestion"]].update_metadata( + vz.Metadata( + { + "variant": p["variant"], + "duration": str(p["duration"]), + "iteration": str(i), + "suggestion": str(s), + } + ) + ) + suggestions[p["suggestion"]].complete(final_measurement) + LOG.info(f"Iteration {i} finished") + except KeyboardInterrupt as ex: + study_client.set_state(vz.StudyState.ABORTED) + raise ex + + study_client.set_state(vz.StudyState.COMPLETED) + + for optimal_trial in study_client.optimal_trials(): + trial = optimal_trial.materialize() + LOG.info(trial.parameters.as_dict()) + LOG.info(trial.final_measurement.metrics) + results["optimals"].append( + { + "params": trial.parameters.as_dict(), + "evaluation": { + k: v.value for k, v in trial.final_measurement.metrics.items() + }, + "variant": f"{args.experiment}/{trial.metadata.get_or_error('variant')}", + "time": float(trial.metadata.get_or_error("duration")), + } + ) + return results + + +def initialize_parser() -> argparse.ArgumentParser: + """ + Creates parser with required arguments. + + Returns + ------- + argparse.ArgumentParser + Preared parser + """ + parser = argparse.ArgumentParser() + add_common_args(parser) + parser.add_argument( + "--experiment", + type=str, + metavar="", + default=f"test-{DATE}", + help="Experiment name. This parameter is used to prefix the" + " FLOW_VARIANT and as the Vizier study ID.", + ) + parser.add_argument( + "--orfs", + type=Path, + default=ORFS, + help="Path to the OpenROAD-flow-scripts repository", + ) + parser.add_argument( + "--results", + type=Path, + default="results.json", + help="Path where JSON file with results will be saved", + ) + parser.add_argument( + "-a", + "--algorithm", + type=str, + choices=[ + "GAUSSIAN_PROCESS_BANDIT", + "RANDOM_SEARCH", + "QUASI_RANDOM_SEARCH", + "GRID_SEARCH", + "SHUFFLED_GRID_SEARCH", + "NSGA2", + ], + help="Algorithm for the optimization engine", + default="NSGA2", + ) + available_metrics = list(METRIC_TO_GOAL.keys()) + parser.add_argument( + "-m", + "--use-metrics", + nargs="+", + choices=available_metrics, + default=available_metrics, + help="Metrics to optimize", + ) + parser.add_argument( + "-i", + "--iterations", + type=int, + help="Max iteration count for the optimization engine", + default=2, + ) + parser.add_argument( + "-s", + "--suggestions", + type=int, + nargs="+", + help="Suggestion count per iteration of the optimization engine", + default=[5], + ) + parser.add_argument( + "-w", + "--workers", + default=2, + help="Number of parallel workers", + type=int, + ) + vizier_server_args = parser.add_mutually_exclusive_group() + vizier_server_args.add_argument( + "--use-existing-server", + type=str, + help="Address of the running Vizier server", + default=None, + ) + start_server_args = vizier_server_args.add_argument_group("Local server") + start_server_args.add_argument( + "--server-host", + type=str, + help="Spawn Vizier server with given host", + default=None, + ) + start_server_args.add_argument( + "--server-port", + type=str, + help="Spawn Vizier server with given port", + default=None, + ) + start_server_args.add_argument( + "--server-db", + type=str, + help="Path to the Vizier server's database", + default=None, + ) + return parser + + +def run_vizier(): + """ + Entrypoint for Vizier optimization. + + Parses arguments and config, prepares Vizier server, + runs optimization and saves the results. + """ + parser = initialize_parser() + args = parser.parse_args() + + LOG.setLevel([logging.ERROR, logging.WARN, logging.INFO][args.verbose]) + + if args.algorithm == "GAUSSIAN_PROCESS_BANDIT" and any( + s > 1 for s in args.suggestions + ): + LOG.error( + "GAUSSIAN_PROCESS_BANDIT does not support batch operation, please set suggestions to 1" + ) + exit(1) + + args.results = args.results.absolute() + args.mode = "vizier" + args.suggestions += [ + args.suggestions[-1] for _ in range(args.iterations - len(args.suggestions)) + ] + if args.to_stage is None: + args.to_stage = "" + + config, sdc_file, fr_file = read_config(args.config, "vizier", args.algorithm) + args.sdc_file = sdc_file + args.fr_file = fr_file + + orfs_flow_dir = str(args.orfs / "flow") + os.chdir(orfs_flow_dir) + install_path = Path("../tools/install").absolute() + + server_endpoint = None + if args.server_host: + # Start Vizier server + server_database = args.server_db if args.server_db else service.SQL_LOCAL_URL + server = servers.DefaultVizierServer( + host=args.server_host, database_url=server_database, port=args.server_port + ) + LOG.info(f"Started Vizier Server at: {server.endpoint}") + LOG.info(f"SQL database file located at: {server._database_url}") + server_endpoint = server.endpoint + if args.use_existing_server: + server_endpoint = args.use_existing_server + + results = main(args, config, install_path, server_endpoint) + with args.results.open("w") as fd: + json.dump(results, fd) + print(f"Results saved to {args.results}") + + +if __name__ == "__main__": + run_vizier()