From aedcc1b8a24e66f0cc55049912420b0951392248 Mon Sep 17 00:00:00 2001 From: Olivier Sprangers Date: Mon, 14 Oct 2024 09:15:33 +0200 Subject: [PATCH 1/4] fix_conformal_issues --- nbs/mint.json | 3 ++- nbs/utils.ipynb | 3 ++- neuralforecast/utils.py | 3 ++- settings.ini | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nbs/mint.json b/nbs/mint.json index 8ead3fc35..529e1a749 100644 --- a/nbs/mint.json +++ b/nbs/mint.json @@ -63,7 +63,8 @@ "group":"Probabilistic Forecasting", "pages":[ "docs/tutorials/uncertainty_quantification.html", - "docs/tutorials/longhorizon_probabilistic.html" + "docs/tutorials/longhorizon_probabilistic.html", + "docs/tutorials/conformal_prediction.html" ] }, { diff --git a/nbs/utils.ipynb b/nbs/utils.ipynb index e202c9adc..e4e5e5873 100644 --- a/nbs/utils.ipynb +++ b/nbs/utils.ipynb @@ -627,7 +627,8 @@ " cuts = [alpha / 200 for alpha in reversed(alphas)]\n", " cuts.extend(1 - alpha / 200 for alpha in alphas)\n", " for model in model_names:\n", - " scores = cs_df[model].to_numpy().reshape(cs_n_windows, n_series, horizon)\n", + " scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)\n", + " scores = scores.transpose(1, 0, 2)\n", " # restrict scores to horizon\n", " scores = scores[:,:,:horizon]\n", " mean = fcst_df[model].to_numpy().reshape(1, n_series, -1)\n", diff --git a/neuralforecast/utils.py b/neuralforecast/utils.py index 9d15ce1ca..6d3ee091e 100644 --- a/neuralforecast/utils.py +++ b/neuralforecast/utils.py @@ -502,7 +502,8 @@ def add_conformal_distribution_intervals( cuts = [alpha / 200 for alpha in reversed(alphas)] cuts.extend(1 - alpha / 200 for alpha in alphas) for model in model_names: - scores = cs_df[model].to_numpy().reshape(cs_n_windows, n_series, horizon) + scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon) + scores = scores.transpose(1, 0, 2) # restrict scores to horizon scores = scores[:, :, :horizon] mean = fcst_df[model].to_numpy().reshape(1, n_series, -1) diff --git a/settings.ini b/settings.ini index 14a2eedaa..3bdb8a5b9 100644 --- a/settings.ini +++ b/settings.ini @@ -15,7 +15,7 @@ language = English custom_sidebar = True license = apache2 status = 2 -requirements = coreforecast>=0.0.6 fsspec numpy>=1.21.6 pandas>=1.3.5 torch>=2.0.0 pytorch-lightning>=2.0.0 ray[tune]>=2.2.0 optuna utilsforecast>=0.0.25 +requirements = coreforecast>=0.0.6 fsspec numpy>=1.21.6 pandas>=1.3.5 torch>=2.0.0 pytorch-lightning>=2.0.0 ray[tune]>=2.2.0 optuna utilsforecast>=0.2.3 spark_requirements = fugue pyspark>=3.5 aws_requirements = fsspec[s3] dev_requirements = black gitpython hyperopt matplotlib mypy nbdev==2.3.25 polars pre-commit pyarrow ruff s3fs transformers From 67cbbe5f36f7b04e6cd2c3231a7cc9ef21f29db9 Mon Sep 17 00:00:00 2001 From: Olivier Sprangers Date: Tue, 15 Oct 2024 10:36:48 +0200 Subject: [PATCH 2/4] rerun_tutorial --- .../tutorials/20_conformal_prediction.ipynb | 251 ++---------------- 1 file changed, 29 insertions(+), 222 deletions(-) diff --git a/nbs/docs/tutorials/20_conformal_prediction.ipynb b/nbs/docs/tutorials/20_conformal_prediction.ipynb index d781c668e..d64424743 100644 --- a/nbs/docs/tutorials/20_conformal_prediction.ipynb +++ b/nbs/docs/tutorials/20_conformal_prediction.ipynb @@ -28,7 +28,18 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\ospra\\miniconda3\\envs\\neuralforecast\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "2024-10-15 10:35:57,968\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n", + "2024-10-15 10:35:58,090\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n" + ] + } + ], "source": [ "import logging\n", "import os\n", @@ -104,200 +115,16 @@ ] }, { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f4e4d84cd6a44714adb9be9ab9c030e5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Sanity Checking: | | 0/? [00:00" ] From a9b78a6c5d20c85a7e384692edb07fddcab72930 Mon Sep 17 00:00:00 2001 From: Olivier Sprangers Date: Tue, 15 Oct 2024 10:37:23 +0200 Subject: [PATCH 3/4] rerun_tutorial --- .../tutorials/20_conformal_prediction.ipynb | 47 ++----------------- 1 file changed, 3 insertions(+), 44 deletions(-) diff --git a/nbs/docs/tutorials/20_conformal_prediction.ipynb b/nbs/docs/tutorials/20_conformal_prediction.ipynb index d64424743..105cfc03b 100644 --- a/nbs/docs/tutorials/20_conformal_prediction.ipynb +++ b/nbs/docs/tutorials/20_conformal_prediction.ipynb @@ -28,18 +28,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Users\\ospra\\miniconda3\\envs\\neuralforecast\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n", - "2024-10-15 10:35:57,968\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n", - "2024-10-15 10:35:58,090\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n" - ] - } - ], + "outputs": [], "source": [ "import logging\n", "import os\n", @@ -105,28 +94,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Seed set to 1\n", - "Seed set to 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 99: 100%|██████████| 1/1 [00:00<00:00, 44.93it/s, v_num=508, train_loss_step=0.138, train_loss_epoch=0.138] \n", - "Predicting DataLoader 0: 100%|██████████| 1/1 [00:00<00:00, 150.75it/s]\n", - "Epoch 99: 100%|██████████| 1/1 [00:00<00:00, 48.70it/s, v_num=510, train_loss_step=2.930, train_loss_epoch=2.930] \n", - "Predicting DataLoader 0: 100%|██████████| 1/1 [00:00<00:00, 32.97it/s]\n", - "Epoch 99: 100%|██████████| 1/1 [00:00<00:00, 49.39it/s, v_num=512, train_loss_step=0.106, train_loss_epoch=0.106] \n", - "Epoch 99: 100%|██████████| 1/1 [00:00<00:00, 48.55it/s, v_num=513, train_loss_step=3.220, train_loss_epoch=3.220] \n" - ] - } - ], + "outputs": [], "source": [ "horizon = 12\n", "input_size = 24\n", @@ -152,16 +120,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Predicting DataLoader 0: 100%|██████████| 1/1 [00:00<00:00, 197.24it/s]\n", - "Predicting DataLoader 0: 100%|██████████| 1/1 [00:00<00:00, 194.88it/s]\n" - ] - } - ], + "outputs": [], "source": [ "preds = nf.predict(futr_df=AirPassengersPanel_test, level=[90])" ] From 5d2ea22934ffc5f05b7957a6b5cbab76d4d3402c Mon Sep 17 00:00:00 2001 From: Olivier Sprangers Date: Tue, 15 Oct 2024 20:22:51 +0200 Subject: [PATCH 4/4] fix_other_conformal_method --- nbs/utils.ipynb | 3 ++- neuralforecast/utils.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nbs/utils.ipynb b/nbs/utils.ipynb index e4e5e5873..5b056c144 100644 --- a/nbs/utils.ipynb +++ b/nbs/utils.ipynb @@ -671,7 +671,8 @@ " cuts = [lv / 100 for lv in level]\n", " for model in model_names:\n", " mean = fcst_df[model].to_numpy().ravel()\n", - " scores = cs_df[model].to_numpy().reshape(cs_n_windows, n_series, horizon)\n", + " scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon)\n", + " scores = scores.transpose(1, 0, 2)\n", " # restrict scores to horizon\n", " scores = scores[:,:,:horizon]\n", " quantiles = np.quantile(\n", diff --git a/neuralforecast/utils.py b/neuralforecast/utils.py index 6d3ee091e..4a272dfcb 100644 --- a/neuralforecast/utils.py +++ b/neuralforecast/utils.py @@ -539,7 +539,8 @@ def add_conformal_error_intervals( cuts = [lv / 100 for lv in level] for model in model_names: mean = fcst_df[model].to_numpy().ravel() - scores = cs_df[model].to_numpy().reshape(cs_n_windows, n_series, horizon) + scores = cs_df[model].to_numpy().reshape(n_series, cs_n_windows, horizon) + scores = scores.transpose(1, 0, 2) # restrict scores to horizon scores = scores[:, :, :horizon] quantiles = np.quantile(