autogluon · Innixma · Oct 5, 2023 · Oct 4, 2023 · Oct 4, 2023 · geoalgo
diff --git a/autogluon_zeroshot/repository/evaluation_repository.py b/autogluon_zeroshot/repository/evaluation_repository.py
@@ -29,7 +29,8 @@ def __init__(
         self._tabular_predictions: TabularModelPredictions = tabular_predictions
         self._zeroshot_context: ZeroshotSimulatorContext = zeroshot_context
         self._ground_truth: dict = ground_truth
-        assert all(x in self._tid_to_name for x in self._tabular_predictions.datasets)
+        if self._tabular_predictions is not None:
+            assert all(x in self._tid_to_name for x in self._tabular_predictions.datasets)
 
     def to_zeroshot(self) -> repository.EvaluationRepositoryZeroshot:
         """
@@ -294,6 +295,10 @@ def _construct_single_best_config_scorer(self, **kwargs) -> SingleBestConfigScor
         )
         return config_scorer
 
+    @classmethod
+    def from_context(cls, version: str = None, lazy_format: bool = True):
+        return load(version=version, lazy_format=lazy_format)
+
 
 def load(version: str = None, lazy_format=True) -> EvaluationRepository:
     from autogluon_zeroshot.contexts import get_context

diff --git a/data/results/all/openml_ag_2022_09_30_cat_zs_models.csv b/data/results/all/openml_ag_2022_09_30_cat_zs_models.csv
diff --git a/data/results/all/openml_ag_2022_09_30_gbm_zs_models.csv b/data/results/all/openml_ag_2022_09_30_gbm_zs_models.csv
diff --git a/data/results/all/openml_ag_2022_10_02_zs_models.csv b/data/results/all/openml_ag_2022_10_02_zs_models.csv
diff --git a/data/results/all/results_ranked_all.csv b/data/results/all/results_ranked_all.csv
diff --git a/data/results/all/results_ranked_by_dataset_all.csv b/data/results/all/results_ranked_by_dataset_all.csv
diff --git a/data/results/all/results_ranked_by_dataset_valid.csv b/data/results/all/results_ranked_by_dataset_valid.csv
diff --git a/data/results/all/results_ranked_valid.csv b/data/results/all/results_ranked_valid.csv
diff --git a/data/results/all_v3/openml_ag_2022_10_13_zs_models.parquet b/data/results/all_v3/openml_ag_2022_10_13_zs_models.parquet
diff --git a/data/results/all_v3/results_ranked_by_dataset_valid.parquet b/data/results/all_v3/results_ranked_by_dataset_valid.parquet
diff --git a/data/results/all_v3/results_ranked_valid.csv b/data/results/all_v3/results_ranked_valid.csv
diff --git a/data/results/automl/results_ranked_by_dataset_valid.csv b/data/results/automl/results_ranked_by_dataset_valid.csv
diff --git a/data/results/automl/results_ranked_valid.csv b/data/results/automl/results_ranked_valid.csv
diff --git a/data/results/bagged/all/results_ranked_by_dataset_valid.parquet b/data/results/bagged/all/results_ranked_by_dataset_valid.parquet
diff --git a/data/results/bagged/all/results_ranked_valid.csv b/data/results/bagged/all/results_ranked_valid.csv
diff --git a/data/results/bagged/openml_ag_2022_12_11_zs_models.parquet b/data/results/bagged/openml_ag_2022_12_11_zs_models.parquet
diff --git a/data/results/bagged/small_30/results_ranked_by_dataset_valid.parquet b/data/results/bagged/small_30/results_ranked_by_dataset_valid.parquet
diff --git a/data/results/bagged/small_30/results_ranked_valid.csv b/data/results/bagged/small_30/results_ranked_valid.csv
diff --git a/data/results/lightgbm/openml_ag_2022_09_30_gbm_zs_models.csv b/data/results/lightgbm/openml_ag_2022_09_30_gbm_zs_models.csv
diff --git a/data/results/lightgbm/results_ranked_by_dataset_valid.csv b/data/results/lightgbm/results_ranked_by_dataset_valid.csv
diff --git a/data/results/lightgbm/results_ranked_valid.csv b/data/results/lightgbm/results_ranked_valid.csv
diff --git a/scripts/simulate/all_v3/run_simulate_zs_ensemble.py b/scripts/simulate/all_v3/run_simulate_zs_ensemble.py
diff --git a/scripts/simulate/all_v3/run_simulate_zs_single_best.py b/scripts/simulate/all_v3/run_simulate_zs_single_best.py
diff --git a/scripts/simulate/bagged/run_inspect_overfit.py b/scripts/simulate/bagged/run_inspect_overfit.py
diff --git a/scripts/simulate/bagged/run_simulate_zs_ensemble_bagged.py b/scripts/simulate/bagged/run_simulate_zs_ensemble_bagged.py
diff --git a/scripts/simulate/bagged/run_simulate_zs_single_best_bagged.py b/scripts/simulate/bagged/run_simulate_zs_single_best_bagged.py
diff --git a/...simulate/bagged_289/run_download_input.py → scripts/simulate/run_download_input.py b/...simulate/bagged_289/run_download_input.py → scripts/simulate/run_download_input.py
@@ -2,7 +2,7 @@
 
 
 if __name__ == '__main__':
-    context_name = 'BAG_D244_F10_C608_FULL'  # The context you want to download
+    context_name = 'BAG_D244_F3_C1416_small'  # The context you want to download
     dry_run = True  # Set False to download files
     include_zs = True  # Set False to only download files necessary for SingleBest (skip predict proba files)
 

diff --git a/...imulate/bagged_289/run_inspect_overfit.py → scripts/simulate/run_inspect_overfit.py b/...imulate/bagged_289/run_inspect_overfit.py → scripts/simulate/run_inspect_overfit.py
@@ -3,19 +3,16 @@
 from autogluon_zeroshot.simulation.sim_runner import run_zs_simulation_debug
 
 
+# TODO: Refactor to use EvaluationRepository, this is old code
 if __name__ == '__main__':
-    # context_name = 'BAG_D244_F10_C608_FULL'
-    context_name = 'BAG_D279_F10_C608_FULL'
+    context_name = 'BAG_D244_F3_C1416_small'
     benchmark_context = get_context(context_name)
     zsc, configs_full, zeroshot_pred_proba, zeroshot_gt = benchmark_context.load()
     zsc.print_info()
 
     # NOTE: For speed of simulation, it is recommended backend='ray'
     backend = 'ray'
-
-    # configs = get_configs_small()
     configs = None
-
     datasets = zsc.get_dataset_folds()
     # datasets = zsc.get_dataset_folds(problem_type=['binary', 'multiclass'])
 

diff --git a/...ed_289/run_simulate_zs_ensemble_bagged.py → ...mulate/run_simulate_zs_ensemble_bagged.py b/...ed_289/run_simulate_zs_ensemble_bagged.py → ...mulate/run_simulate_zs_ensemble_bagged.py
@@ -6,7 +6,7 @@
 
 
 if __name__ == '__main__':
-    subcontext_name = 'BAG_D244_F10_C608_FULL'
+    subcontext_name = 'BAG_D244_F3_C1416_small'
     results_cv, repo = run_zs_sim_end_to_end(subcontext_name=subcontext_name,
                                              config_scorer_type='ensemble')
 

diff --git a/...289/run_simulate_zs_single_best_bagged.py → ...ate/run_simulate_zs_single_best_bagged.py b/...289/run_simulate_zs_single_best_bagged.py → ...ate/run_simulate_zs_single_best_bagged.py
@@ -6,7 +6,7 @@
 
 
 if __name__ == '__main__':
-    subcontext_name = 'BAG_D244_F10_C608_FULL'
+    subcontext_name = 'BAG_D244_F3_C1416_small'
     results_cv, repo = run_zs_sim_end_to_end(subcontext_name=subcontext_name,
                                              config_scorer_type='single')