Simplify the code base; saved fitted rf_plus; add other FI

Yu-Group · May 22, 2024 · 3430b61 · 3430b61
1 parent a7070d2
commit 3430b61
Show file tree

Hide file tree

Showing 29 changed files with 106,811 additions and 12,392 deletions.
diff --git a/.gitignore b/.gitignore
@@ -45,3 +45,5 @@ experiments/*.txt
 _site
 
 temp.ipynb
+**.png
+data
diff --git a/feature_importance/01_ablation_classification_script.sh b/feature_importance/01_ablation_classification_script.sh
@@ -4,7 +4,7 @@
 #SBATCH --partition=yugroup
 
 source activate mdi
-command="01_run_ablation_classification.py --nreps 1 --config mdi_local.real_data_classification --split_seed 1 --ignore_cache --create_rmd --result_name diabetes_classification"
-
+command="01_run_ablation_classification.py --nreps 1 --config mdi_local.real_data_classification --split_seed ${1} --ignore_cache --create_rmd --result_name diabetes_simplify"
+# command="01_run_ablation_classification.py --nreps 1 --config mdi_local.real_data_classification --split_seed ${1} --ignore_cache --create_rmd --result_name Enhancer --ablate_features 20"
 # Execute the command
 python $command
diff --git a/feature_importance/01_ablation_regression_script.sh b/feature_importance/01_ablation_regression_script.sh
@@ -4,7 +4,8 @@
 #SBATCH --partition=yugroup
 
 source activate mdi
-command="01_run_ablation_regression.py --nreps 1 --config mdi_local.real_data_regression --split_seed ${1} --ignore_cache --create_rmd --result_name diabetes_regr_new"
+command="01_run_ablation_regression.py --nreps 1 --config mdi_local.real_data_regression --split_seed ${1} --ignore_cache --create_rmd --result_name diabetes_test_new"
+#command="01_run_ablation_regression.py --nreps 1 --config mdi_local.real_data_regression --split_seed ${1} --ignore_cache --create_rmd --result_name CCLE_AZD0530_new --ablate_features 20"
 
 # Execute the command
 python $command
diff --git a/feature_importance/01_ablation_script.sh → ...re_importance/01_ablation_script_class.sh b/feature_importance/01_ablation_script.sh → ...re_importance/01_ablation_script_class.sh
@@ -2,7 +2,7 @@
 
 slurm_script="01_ablation_classification_script.sh"
 
-for rep in {1..5}
+for rep in {1..2}
 do
     sbatch $slurm_script $rep  # Submit SLURM job using the specified script
 done
diff --git a/feature_importance/01_ablation_script_regr.sh b/feature_importance/01_ablation_script_regr.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+slurm_script="01_ablation_regression_script.sh"
+
+for rep in {1..2}
+do
+    sbatch $slurm_script $rep  # Submit SLURM job using the specified script
+done
diff --git a/feature_importance/01_run_ablation_classification.py b/feature_importance/01_run_ablation_classification.py
diff --git a/feature_importance/01_run_ablation_regression.py b/feature_importance/01_run_ablation_regression.py
diff --git a/feature_importance/diabetes_classification_test.png b/feature_importance/diabetes_classification_test.png
diff --git a/feature_importance/diabetes_classification_test_subset_1.png b/feature_importance/diabetes_classification_test_subset_1.png
diff --git a/feature_importance/diabetes_classification_test_subset_2.png b/feature_importance/diabetes_classification_test_subset_2.png
diff --git a/feature_importance/diabetes_classification_train.png b/feature_importance/diabetes_classification_train.png
diff --git a/feature_importance/diabetes_regression_test.png b/feature_importance/diabetes_regression_test.png
diff --git a/feature_importance/diabetes_regression_test_subset_1.png b/feature_importance/diabetes_regression_test_subset_1.png
diff --git a/feature_importance/diabetes_regression_test_subset_2.png b/feature_importance/diabetes_regression_test_subset_2.png
diff --git a/feature_importance/diabetes_regression_train.png b/feature_importance/diabetes_regression_train.png
diff --git a/feature_importance/fi_config/mdi_local/real_data_classification/dgp.py b/feature_importance/fi_config/mdi_local/real_data_classification/dgp.py
@@ -9,13 +9,36 @@
     "data_name": "diabetes",
     "sample_row_n": None
 }
+# X_PARAMS_DICT = {
+#     "source": "imodels",
+#     "data_name": "juvenile",
+#     "sample_row_n": None
+# }
+
+# X_PARAMS_DICT = {
+#     "source": "csv",
+#     "file_path": "/accounts/projects/binyu/zhongyuan_liang/local_MDI+/imodels-experiments/feature_importance/data/Enhancer/X_enhancer_cleaned.csv",
+#     "sample_row_n": 2000,
+#     "normalize": False
+# }
 
 Y_DGP = sample_real_data_y
 Y_PARAMS_DICT = {
     "source": "imodels",
     "data_name": "diabetes"
 }
+# Y_PARAMS_DICT = {
+#     "source": "imodels",
+#     "data_name": "juvenile"
+# }
+
+# Y_PARAMS_DICT = {
+#     "source": "csv",
+#     "file_path": "/accounts/projects/binyu/zhongyuan_liang/local_MDI+/imodels-experiments/feature_importance/data/Enhancer/y_enhancer.csv",
+#     "sample_row_n": 2000
+# }
+
 
 # vary one parameter
-VARY_PARAM_NAME = "sample_row_n"
+VARY_PARAM_NAME = "normalize"
 VARY_PARAM_VALS = {"keep_all_rows": None}
diff --git a/feature_importance/fi_config/mdi_local/real_data_classification/models.py b/feature_importance/fi_config/mdi_local/real_data_classification/models.py
@@ -14,11 +14,13 @@
 ]
 
 FI_ESTIMATORS = [
-    [FIModelConfig('TreeSHAP_RF', tree_shap_evaluation_RF, model_type='tree', splitting_strategy = "train-test")],
-    [FIModelConfig('LFI_fit_on_inbag_RF', LFI_evaluation_RF_MDI_classification, model_type='tree', splitting_strategy = "train-test", ascending = False, other_params={"include_raw":False, "fit_on":"inbag", "prediction_model": Ridge(alpha=1e-6)})],
-    [FIModelConfig('LFI_fit_on_OOB_RF', LFI_evaluation_RF_OOB, model_type='tree', splitting_strategy = "train-test",  ascending = False, other_params={"fit_on":"oob"})],
-    [FIModelConfig('LFI_evaluate_on_all_RF_plus', LFI_evaluation_RF_plus, model_type='tree', splitting_strategy = "train-test", ascending = False)],
-    [FIModelConfig('LFI_evaluate_on_oob_RF_plus', LFI_evaluation_RF_plus_OOB, model_type='tree', splitting_strategy = "train-test", ascending = False)],
-    [FIModelConfig('Kernel_SHAP_RF_plus', kernel_shap_evaluation_RF_plus, model_type='tree', splitting_strategy = "train-test")],
-    [FIModelConfig('LIME_RF_plus', lime_evaluation_RF_plus, model_type='tree', splitting_strategy = "train-test")],
+    [FIModelConfig('TreeSHAP_RF', tree_shap_evaluation_RF, model_type='tree', base_model="RF", splitting_strategy = "train-test")],
+    [FIModelConfig('LFI_fit_on_inbag_RFPlus', LFI_evaluation_RFPlus_inbag, model_type='tree', base_model="RFPlus_inbag", splitting_strategy = "train-test", ascending = False)],
+    [FIModelConfig('LFI_fit_on_OOB_RFPlus', LFI_evaluation_RFPlus_oob, model_type='tree', base_model="RFPlus_oob", splitting_strategy = "train-test",  ascending = False)],
+    [FIModelConfig('LFI_fit_on_all_evaluate_on_all_RFPlus', LFI_evaluation_RFPlus_all, model_type='tree', base_model="RFPlus_default", splitting_strategy = "train-test", ascending = False)],
+    [FIModelConfig('LFI_fit_on_all_evaluate_on_oob_RFPlus', LFI_evaluation_RFPlus_oob, model_type='tree', base_model="RFPlus_default", splitting_strategy = "train-test", ascending = False)],
+    [FIModelConfig('Kernel_SHAP_RF_plus', kernel_shap_evaluation_RF_plus, model_type='tree', base_model="RFPlus_default", splitting_strategy = "train-test")],
+    [FIModelConfig('LIME_RF_plus', lime_evaluation_RF_plus, model_type='tree', base_model="RFPlus_default", splitting_strategy = "train-test")],
+    [FIModelConfig('Random', None, model_type='tree', base_model="None", splitting_strategy = "train-test")],
+    [FIModelConfig('Oracle_test_RFPlus', LFI_evaluation_oracle_RF_plus, base_model="RFPlus_default", model_type='tree', splitting_strategy = "train-test", ascending = False)],
 ]
diff --git a/feature_importance/fi_config/mdi_local/real_data_regression/dgp.py b/feature_importance/fi_config/mdi_local/real_data_regression/dgp.py
@@ -16,7 +16,12 @@
 # }
 # X_PARAMS_DICT = {
 #     "source": "openml",
-#     "task_id": 359946,
+#     "task_id": 361236,
+#     "sample_row_n": None
+# }
+# X_PARAMS_DICT = {
+#     "source": "csv",
+#     "file_path": "/accounts/projects/binyu/zhongyuan_liang/local_MDI+/imodels-experiments/feature_importance/data/CCLE/X_ccle_rnaseq_cleaned_filtered5000.csv",
 #     "sample_row_n": None
 # }
 
@@ -31,7 +36,12 @@
 # }
 # Y_PARAMS_DICT = {
 #     "source": "openml",
-#     "task_id": 359946
+#     "task_id": 361236
+# }
+
+# Y_PARAMS_DICT = {
+#     "source": "csv",
+#     "file_path": "/accounts/projects/binyu/zhongyuan_liang/local_MDI+/imodels-experiments/feature_importance/data/CCLE/y_ccle_rnaseq_AZD0530.csv",
 # }
 
 # vary one parameter

diff --git a/feature_importance/fi_config/mdi_local/real_data_regression/models.py b/feature_importance/fi_config/mdi_local/real_data_regression/models.py
@@ -14,11 +14,13 @@
 ]
 
 FI_ESTIMATORS = [
-    [FIModelConfig('TreeSHAP_RF', tree_shap_evaluation_RF, model_type='tree', splitting_strategy = "train-test")],
-    [FIModelConfig('LFI_fit_on_inbag_RF', LFI_evaluation_RF_MDI, model_type='tree', splitting_strategy = "train-test", ascending = False, other_params={"include_raw":False, "fit_on":"inbag", "prediction_model": Ridge(alpha=1e-6)})],
-    [FIModelConfig('LFI_fit_on_OOB_RF', LFI_evaluation_RF_OOB, model_type='tree', splitting_strategy = "train-test",  ascending = False, other_params={"fit_on":"oob"})],
-    [FIModelConfig('LFI_evaluate_on_all_RF_plus', LFI_evaluation_RF_plus, model_type='tree', splitting_strategy = "train-test", ascending = False)],
-    [FIModelConfig('LFI_evaluate_on_oob_RF_plus', LFI_evaluation_RF_plus_OOB, model_type='tree', splitting_strategy = "train-test", ascending = False)],
-    [FIModelConfig('Kernel_SHAP_RF_plus', kernel_shap_evaluation_RF_plus, model_type='tree', splitting_strategy = "train-test")],
-    [FIModelConfig('LIME_RF_plus', lime_evaluation_RF_plus, model_type='tree', splitting_strategy = "train-test")],
+    [FIModelConfig('TreeSHAP_RF', tree_shap_evaluation_RF, model_type='tree', base_model="RF", splitting_strategy = "train-test")],
+    [FIModelConfig('LFI_fit_on_inbag_RFPlus', LFI_evaluation_RFPlus_inbag, model_type='tree', base_model="RFPlus_inbag", splitting_strategy = "train-test", ascending = False)],
+    [FIModelConfig('LFI_fit_on_OOB_RFPlus', LFI_evaluation_RFPlus_oob, model_type='tree', base_model="RFPlus_oob", splitting_strategy = "train-test",  ascending = False)],
+    [FIModelConfig('LFI_fit_on_all_evaluate_on_all_RFPlus', LFI_evaluation_RFPlus_all, model_type='tree', base_model="RFPlus_default", splitting_strategy = "train-test", ascending = False)],
+    [FIModelConfig('LFI_fit_on_all_evaluate_on_oob_RFPlus', LFI_evaluation_RFPlus_oob, model_type='tree', base_model="RFPlus_default", splitting_strategy = "train-test", ascending = False)],
+    [FIModelConfig('Kernel_SHAP_RF_plus', kernel_shap_evaluation_RF_plus, model_type='tree', base_model="RFPlus_default", splitting_strategy = "train-test")],
+    [FIModelConfig('LIME_RF_plus', lime_evaluation_RF_plus, model_type='tree', base_model="RFPlus_default", splitting_strategy = "train-test")],
+    [FIModelConfig('Random', None, model_type='tree', base_model="None", splitting_strategy = "train-test")],
+    [FIModelConfig('Oracle_test_RFPlus', LFI_evaluation_oracle_RF_plus, base_model="RFPlus_default", model_type='tree', splitting_strategy = "train-test", ascending = False)],
 ]