restore base auto tests (#774)

* single command setup * trigger ci * restore versions and tests * install ray-tune with conda * add hyperopt to env * comment out optuna * trigger ci * comment out ray * try large resource class * try py39 * restore cpus argument * restore py38 * remove missing comments * move ray tune to pip
Nixtla · Oct 5, 2023 · 85f2d25 · 85f2d25
1 parent 341f8b2
commit 85f2d25
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 56 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,9 +8,7 @@ jobs:
       - checkout
       - run:
           name: Install dependencies
-          command: |
-            micromamba install -n base -c conda-forge -y python=3.8
-            micromamba update -n base -f environment.yml
+          command: micromamba install -n base -c conda-forge -y python=3.8 -f environment.yml
       - run:
           name: Run nbdev tests
           command: |
@@ -26,9 +24,7 @@ jobs:
       - checkout
       - run:
           name: Install dependencies
-          command: |
-            micromamba install -n base -c conda-forge -y python=3.8
-            micromamba update -n base -f environment.yml
+          command: micromamba install -n base -c conda-forge -y python=3.8 -f environment.yml
       - run:
           name: Run model performance tests
           command: |
@@ -52,9 +48,7 @@ jobs:
       - checkout
       - run:
           name: Install dependencies
-          command: |
-            micromamba install -n base -c conda-forge -y python=3.8
-            micromamba update -n base -f environment.yml
+          command: micromamba install -n base -c conda-forge -y python=3.8 -f environment.yml
       - run:
           name: Run model performance tests
           command: |

diff --git a/environment.yml b/environment.yml
@@ -3,6 +3,7 @@ channels:
   - conda-forge
   - pytorch
 dependencies:
+  - hyperopt
   - jupyterlab
   - matplotlib
   - numba
@@ -16,4 +17,4 @@ dependencies:
   - pip:
     - nbdev
     - black
-    - "ray[tune]==2.6.3"
+    - "ray[tune]>=2.2.0"
diff --git a/nbs/common.base_auto.ipynb b/nbs/common.base_auto.ipynb
@@ -556,17 +556,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# config = {\n",
-    "#     \"hidden_size\": tune.choice([512]),\n",
-    "#     \"num_layers\": tune.choice([3, 4]),\n",
-    "#     \"input_size\": 12,\n",
-    "#     \"max_steps\": 10,\n",
-    "#     \"val_check_steps\": 5\n",
-    "# }\n",
-    "# auto = BaseAuto(h=12, loss=MAE(), valid_loss=MSE(), cls_model=MLP, config=config, num_samples=2) # cpus=1, gpus=0\n",
-    "# auto.fit(dataset=dataset)\n",
-    "# y_hat = auto.predict(dataset=dataset)\n",
-    "# assert mae(Y_test_df['y'].values, y_hat[:, 0]) < 200"
+    "config = {\n",
+    "    \"hidden_size\": tune.choice([512]),\n",
+    "    \"num_layers\": tune.choice([3, 4]),\n",
+    "    \"input_size\": 12,\n",
+    "    \"max_steps\": 10,\n",
+    "    \"val_check_steps\": 5\n",
+    "}\n",
+    "auto = BaseAuto(h=12, loss=MAE(), valid_loss=MSE(), cls_model=MLP, config=config, num_samples=2, cpus=1, gpus=0)\n",
+    "auto.fit(dataset=dataset)\n",
+    "y_hat = auto.predict(dataset=dataset)\n",
+    "assert mae(Y_test_df['y'].values, y_hat[:, 0]) < 200"
    ]
   },
   {
@@ -607,10 +607,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# #| hide\n",
-    "# Y_test_df['AutoMLP'] = y_hat\n",
+    "#| hide\n",
+    "Y_test_df['AutoMLP'] = y_hat\n",
     "\n",
-    "# pd.concat([Y_train_df, Y_test_df]).drop('unique_id', axis=1).set_index('ds').plot()"
+    "pd.concat([Y_train_df, Y_test_df]).drop('unique_id', axis=1).set_index('ds').plot()"
    ]
   },
   {
@@ -620,37 +620,37 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# #| hide\n",
-    "# # Unit tests to guarantee that losses are correctly instantiated\n",
-    "# import pandas as pd\n",
-    "# from neuralforecast.models.mlp import MLP\n",
-    "# from neuralforecast.utils import AirPassengersDF as Y_df\n",
-    "# from neuralforecast.tsdataset import TimeSeriesDataset\n",
-    "# from neuralforecast.losses.pytorch import MAE, MSE\n",
-    "\n",
-    "# Y_train_df = Y_df[Y_df.ds<='1959-12-31'] # 132 train\n",
-    "# Y_test_df = Y_df[Y_df.ds>'1959-12-31']   # 12 test\n",
-    "\n",
-    "# dataset, *_ = TimeSeriesDataset.from_df(Y_train_df)\n",
-    "# config = {\n",
-    "#     \"hidden_size\": tune.choice([512]),\n",
-    "#     \"num_layers\": tune.choice([3, 4]),\n",
-    "#     \"input_size\": 12,\n",
-    "#     \"max_steps\": 1,\n",
-    "#     \"val_check_steps\": 1\n",
-    "# }\n",
-    "\n",
-    "# # Test instantiation\n",
-    "# auto = BaseAuto(h=12, loss=MAE(), valid_loss=MSE(), \n",
-    "#                 cls_model=MLP, config=config, num_samples=2, cpus=1, gpus=0)\n",
-    "# test_eq(str(type(auto.loss)), \"<class 'neuralforecast.losses.pytorch.MAE'>\")\n",
-    "# test_eq(str(type(auto.valid_loss)), \"<class 'neuralforecast.losses.pytorch.MSE'>\")\n",
-    "\n",
-    "# # Test validation default\n",
-    "# auto = BaseAuto(h=12, loss=MSE(), valid_loss=None,\n",
-    "#                 cls_model=MLP, config=config, num_samples=2, cpus=1, gpus=0)\n",
-    "# test_eq(str(type(auto.loss)), \"<class 'neuralforecast.losses.pytorch.MSE'>\")\n",
-    "# test_eq(str(type(auto.valid_loss)), \"<class 'neuralforecast.losses.pytorch.MSE'>\")"
+    "#| hide\n",
+    "# Unit tests to guarantee that losses are correctly instantiated\n",
+    "import pandas as pd\n",
+    "from neuralforecast.models.mlp import MLP\n",
+    "from neuralforecast.utils import AirPassengersDF as Y_df\n",
+    "from neuralforecast.tsdataset import TimeSeriesDataset\n",
+    "from neuralforecast.losses.pytorch import MAE, MSE\n",
+    "\n",
+    "Y_train_df = Y_df[Y_df.ds<='1959-12-31'] # 132 train\n",
+    "Y_test_df = Y_df[Y_df.ds>'1959-12-31']   # 12 test\n",
+    "\n",
+    "dataset, *_ = TimeSeriesDataset.from_df(Y_train_df)\n",
+    "config = {\n",
+    "    \"hidden_size\": tune.choice([512]),\n",
+    "    \"num_layers\": tune.choice([3, 4]),\n",
+    "    \"input_size\": 12,\n",
+    "    \"max_steps\": 1,\n",
+    "    \"val_check_steps\": 1\n",
+    "}\n",
+    "\n",
+    "# Test instantiation\n",
+    "auto = BaseAuto(h=12, loss=MAE(), valid_loss=MSE(), \n",
+    "                cls_model=MLP, config=config, num_samples=2, cpus=1, gpus=0)\n",
+    "test_eq(str(type(auto.loss)), \"<class 'neuralforecast.losses.pytorch.MAE'>\")\n",
+    "test_eq(str(type(auto.valid_loss)), \"<class 'neuralforecast.losses.pytorch.MSE'>\")\n",
+    "\n",
+    "# Test validation default\n",
+    "auto = BaseAuto(h=12, loss=MSE(), valid_loss=None,\n",
+    "                cls_model=MLP, config=config, num_samples=2, cpus=1, gpus=0)\n",
+    "test_eq(str(type(auto.loss)), \"<class 'neuralforecast.losses.pytorch.MSE'>\")\n",
+    "test_eq(str(type(auto.valid_loss)), \"<class 'neuralforecast.losses.pytorch.MSE'>\")"
    ]
   },
   {

diff --git a/settings.ini b/settings.ini
@@ -15,7 +15,7 @@ language = English
 custom_sidebar = True
 license = apache2
 status = 2
-requirements = numpy>=1.21.6 pandas>=1.3.5 torch>=2.0.0 pytorch-lightning>=2.0.0 ray[tune]==2.6.3 optuna utilsforecast>=0.0.6 numba
+requirements = numpy>=1.21.6 pandas>=1.3.5 torch>=2.0.0 pytorch-lightning>=2.0.0 ray[tune]>=2.2.0 optuna utilsforecast>=0.0.6 numba
 dev_requirements = nbdev black mypy flake8 matplotlib hyperopt
 nbs_path = nbs
 doc_path = _docs