examples/Gordo-Workflow-High-Level.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gordo workflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a higher level example of how gordo works.\n",
    "\n",
    "Train a model from a config file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # Tensorflow INFO and WARNING messages are not printed\n",
    "\n",
    "# Temporory suppress warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", module='sklearn.base', category=UserWarning) # sklearn/base.py:446: UserWarning: X does not have valid feature names, but MinMaxScaler was fitted with feature names\n",
    "warnings.filterwarnings(\"ignore\", module='gordo.machine.model.anomaly.diff', category=FutureWarning) # gordo/machine/model/anomaly/diff.py:236: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead\n",
    "from gordo_core.time_series import NotEnoughDataWarning\n",
    "warnings.filterwarnings(\"ignore\", category=NotEnoughDataWarning)\n",
    "\n",
    "import tempfile\n",
    "import yaml\n",
    "from pprint import pprint\n",
    "from dateutil.parser import isoparse\n",
    "\n",
    "from gordo import serializer\n",
    "from gordo.builder import local_build"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define some config file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = \\\n",
    "\"\"\"\n",
    "machines:\n",
    "  - dataset: |\n",
    "      tags:\n",
    "        - SOME-TAG1\n",
    "        - SOME-TAG2\n",
    "      target_tag_list:\n",
    "        - SOME-TAG3\n",
    "        - SOME-TAG4\n",
    "      train_end_date: '2019-03-01T00:00:00+00:00'\n",
    "      train_start_date: '2019-01-01T00:00:00+00:00'\n",
    "      data_provider:\n",
    "        type: RandomDataProvider\n",
    "    metadata: |\n",
    "      information: Some sweet information about the model\n",
    "    model: |\n",
    "      gordo.machine.model.anomaly.diff.DiffBasedAnomalyDetector:\n",
    "        base_estimator:\n",
    "          sklearn.pipeline.Pipeline:\n",
    "            steps:\n",
    "            - sklearn.decomposition.PCA\n",
    "            - sklearn.multioutput.MultiOutputRegressor:\n",
    "                estimator: sklearn.linear_model.LinearRegression\n",
    "    name: crazy-sweet-name\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build model from data and model configs\n",
    "\n",
    "`gordo.builder.local_build` is a generator for building each machine in your string config, much in the same actual deployment on k8s works. \n",
    "\n",
    "Here we'll simply cast it as a list and get the first element, which is a `Tuple[BaseEstimator, dict]` of your model and metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe, metadata = next(local_build(config))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### The trained model/pipeline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>DiffBasedAnomalyDetector(base_estimator=Pipeline(steps=[(&#x27;step_0&#x27;, PCA()),\n",
       "                                                        (&#x27;step_1&#x27;,\n",
       "                                                         MultiOutputRegressor(estimator=LinearRegression()))]))</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DiffBasedAnomalyDetector</label><div class=\"sk-toggleable__content\"><pre>DiffBasedAnomalyDetector(base_estimator=Pipeline(steps=[(&#x27;step_0&#x27;, PCA()),\n",
       "                                                        (&#x27;step_1&#x27;,\n",
       "                                                         MultiOutputRegressor(estimator=LinearRegression()))]))</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">PCA</label><div class=\"sk-toggleable__content\"><pre>PCA()</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">step_1: MultiOutputRegressor</label><div class=\"sk-toggleable__content\"><pre>MultiOutputRegressor(estimator=LinearRegression())</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "DiffBasedAnomalyDetector(base_estimator=Pipeline(steps=[('step_0', PCA()),\n",
       "                                                        ('step_1',\n",
       "                                                         MultiOutputRegressor(estimator=LinearRegression()))]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metadata from the model and build process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset:\n",
      "  additional_tags: null\n",
      "  aggregation_methods: mean\n",
      "  asset: null\n",
      "  data_provider:\n",
      "    max_size: 300\n",
      "    min_size: 100\n",
      "    type: gordo_core.data_providers.providers.RandomDataProvider\n",
      "  default_tag: null\n",
      "  filter_periods: null\n",
      "  interpolation_limit: 48H\n",
      "  interpolation_method: linear_interpolation\n",
      "  known_filter_periods: null\n",
      "  n_samples_threshold: 0\n",
      "  resolution: 10T\n",
      "  row_filter: ''\n",
      "  row_filter_buffer_size: 0\n",
      "  tag_list:\n",
      "  - SOME-TAG1\n",
      "  - SOME-TAG2\n",
      "  target_tag_list:\n",
      "  - SOME-TAG3\n",
      "  - SOME-TAG4\n",
      "  train_end_date: '2019-03-01T00:00:00+00:00'\n",
      "  train_start_date: '2019-01-01T00:00:00+00:00'\n",
      "  type: gordo_core.time_series.TimeSeriesDataset\n",
      "evaluation:\n",
      "  cv_mode: full_build\n",
      "  metrics:\n",
      "  - explained_variance_score\n",
      "  - r2_score\n",
      "  - mean_squared_error\n",
      "  - mean_absolute_error\n",
      "  scoring_scaler: sklearn.preprocessing.MinMaxScaler\n",
      "metadata:\n",
      "  build_metadata:\n",
      "    dataset:\n",
      "      dataset_meta:\n",
      "        row_filter_tags: []\n",
      "        summary_statistics:\n",
      "          SOME-TAG1:\n",
      "            25%: 0.3298567003643994\n",
      "            50%: 0.5203914351688911\n",
      "            75%: 0.7179140879378773\n",
      "            count: 8376.0\n",
      "            max: 0.9923963988886326\n",
      "            mean: 0.5209829767778671\n",
      "            min: 5.535686500857118e-05\n",
      "            std: 0.24174500952086836\n",
      "          SOME-TAG2:\n",
      "            25%: 0.29626022336172775\n",
      "            50%: 0.47227110171791264\n",
      "            75%: 0.7022930594629675\n",
      "            count: 8376.0\n",
      "            max: 0.9992779939221711\n",
      "            mean: 0.4967139973731738\n",
      "            min: 0.0013833499989991394\n",
      "            std: 0.2463834505198278\n",
      "        tag_loading_metadata:\n",
      "          SOME-TAG1:\n",
      "            first_timestamp: 1546314000.0\n",
      "            gaps:\n",
      "              end:\n",
      "              - 1546314000.0\n",
      "              start:\n",
      "              - 1546300800.0\n",
      "            last_timestamp: 1551398400.0\n",
      "            original_length: 198\n",
      "            resampled_length: 8475\n",
      "          SOME-TAG2:\n",
      "            first_timestamp: 1546302000.0\n",
      "            gaps:\n",
      "              end:\n",
      "              - 1546302000.0\n",
      "              start:\n",
      "              - 1546300800.0\n",
      "            last_timestamp: 1551398400.0\n",
      "            original_length: 294\n",
      "            resampled_length: 8495\n",
      "          SOME-TAG3:\n",
      "            first_timestamp: 1546333200.0\n",
      "            gaps:\n",
      "              end:\n",
      "              - 1546333200.0\n",
      "              start:\n",
      "              - 1546300800.0\n",
      "            last_timestamp: 1551398400.0\n",
      "            original_length: 207\n",
      "            resampled_length: 8443\n",
      "          SOME-TAG4:\n",
      "            first_timestamp: 1546327200.0\n",
      "            gaps:\n",
      "              end:\n",
      "              - 1546327200.0\n",
      "              - 1550126400.0\n",
      "              start:\n",
      "              - 1546300800.0\n",
      "              - 1550085600.0\n",
      "            last_timestamp: 1551398400.0\n",
      "            original_length: 110\n",
      "            resampled_length: 8386\n",
      "          aggregate_metadata:\n",
      "            dropped_na_length: 8376\n",
      "            joined_length: 8376\n",
      "          tags:\n",
      "            SOME-TAG1:\n",
      "              name: SOME-TAG1\n",
      "            SOME-TAG2:\n",
      "              name: SOME-TAG2\n",
      "            SOME-TAG3:\n",
      "              name: SOME-TAG3\n",
      "            SOME-TAG4:\n",
      "              name: SOME-TAG4\n",
      "        train_end_date_actual: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "        - 1551398400000000000\n",
      "        - null\n",
      "        - &id001 !!python/object/apply:pytz._UTC []\n",
      "        - 10\n",
      "        train_start_date_actual: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "        - 1546333200000000000\n",
      "        - null\n",
      "        - *id001\n",
      "        - 10\n",
      "        x_hist:\n",
      "          SOME-TAG1: '{\"(-0.00987, 5.5e-05]\":0,\"(5.5e-05, 0.00998]\":5,\"(0.00998, 0.0199]\":12,\"(0.0199,\n",
      "            0.0298]\":16,\"(0.0298, 0.0397]\":19,\"(0.0397, 0.0497]\":27,\"(0.0497, 0.0596]\":37,\"(0.0596,\n",
      "            0.0695]\":42,\"(0.0695, 0.0794]\":39,\"(0.0794, 0.0894]\":39,\"(0.0894, 0.0993]\":37,\"(0.0993,\n",
      "            0.109]\":41,\"(0.109, 0.119]\":43,\"(0.119, 0.129]\":37,\"(0.129, 0.139]\":38,\"(0.139,\n",
      "            0.149]\":49,\"(0.149, 0.159]\":49,\"(0.159, 0.169]\":163,\"(0.169, 0.179]\":58,\"(0.179,\n",
      "            0.189]\":72,\"(0.189, 0.199]\":76,\"(0.199, 0.208]\":80,\"(0.208, 0.218]\":91,\"(0.218,\n",
      "            0.228]\":320,\"(0.228, 0.238]\":71,\"(0.238, 0.248]\":71,\"(0.248, 0.258]\":69,\"(0.258,\n",
      "            0.268]\":71,\"(0.268, 0.278]\":67,\"(0.278, 0.288]\":64,\"(0.288, 0.298]\":64,\"(0.298,\n",
      "            0.308]\":65,\"(0.308, 0.318]\":70,\"(0.318, 0.328]\":75,\"(0.328, 0.337]\":77,\"(0.337,\n",
      "            0.347]\":75,\"(0.347, 0.357]\":81,\"(0.357, 0.367]\":98,\"(0.367, 0.377]\":102,\"(0.377,\n",
      "            0.387]\":103,\"(0.387, 0.397]\":97,\"(0.397, 0.407]\":89,\"(0.407, 0.417]\":117,\"(0.417,\n",
      "            0.427]\":118,\"(0.427, 0.437]\":77,\"(0.437, 0.447]\":72,\"(0.447, 0.457]\":81,\"(0.457,\n",
      "            0.466]\":137,\"(0.466, 0.476]\":167,\"(0.476, 0.486]\":232,\"(0.486, 0.496]\":140,\"(0.496,\n",
      "            0.506]\":100,\"(0.506, 0.516]\":107,\"(0.516, 0.526]\":94,\"(0.526, 0.536]\":97,\"(0.536,\n",
      "            0.546]\":91,\"(0.546, 0.556]\":106,\"(0.556, 0.566]\":99,\"(0.566, 0.576]\":102,\"(0.576,\n",
      "            0.586]\":98,\"(0.586, 0.595]\":93,\"(0.595, 0.605]\":153,\"(0.605, 0.615]\":126,\"(0.615,\n",
      "            0.625]\":109,\"(0.625, 0.635]\":97,\"(0.635, 0.645]\":106,\"(0.645, 0.655]\":100,\"(0.655,\n",
      "            0.665]\":115,\"(0.665, 0.675]\":95,\"(0.675, 0.685]\":110,\"(0.685, 0.695]\":101,\"(0.695,\n",
      "            0.705]\":112,\"(0.705, 0.715]\":97,\"(0.715, 0.724]\":94,\"(0.724, 0.734]\":109,\"(0.734,\n",
      "            0.744]\":132,\"(0.744, 0.754]\":114,\"(0.754, 0.764]\":89,\"(0.764, 0.774]\":92,\"(0.774,\n",
      "            0.784]\":99,\"(0.784, 0.794]\":89,\"(0.794, 0.804]\":93,\"(0.804, 0.814]\":83,\"(0.814,\n",
      "            0.824]\":83,\"(0.824, 0.834]\":78,\"(0.834, 0.844]\":77,\"(0.844, 0.853]\":80,\"(0.853,\n",
      "            0.863]\":68,\"(0.863, 0.873]\":66,\"(0.873, 0.883]\":83,\"(0.883, 0.893]\":92,\"(0.893,\n",
      "            0.903]\":84,\"(0.903, 0.913]\":77,\"(0.913, 0.923]\":80,\"(0.923, 0.933]\":60,\"(0.933,\n",
      "            0.943]\":55,\"(0.943, 0.953]\":41,\"(0.953, 0.963]\":41,\"(0.963, 0.973]\":39,\"(0.973,\n",
      "            0.982]\":21,\"(0.982, 0.992]\":8,\"(0.992, 1.002]\":1}'\n",
      "          SOME-TAG2: '{\"(-0.0086, 0.00138]\":0,\"(0.00138, 0.0114]\":8,\"(0.0114, 0.0213]\":7,\"(0.0213,\n",
      "            0.0313]\":12,\"(0.0313, 0.0413]\":8,\"(0.0413, 0.0513]\":14,\"(0.0513, 0.0613]\":15,\"(0.0613,\n",
      "            0.0712]\":25,\"(0.0712, 0.0812]\":44,\"(0.0812, 0.0912]\":126,\"(0.0912, 0.101]\":33,\"(0.101,\n",
      "            0.111]\":66,\"(0.111, 0.121]\":95,\"(0.121, 0.131]\":71,\"(0.131, 0.141]\":64,\"(0.141,\n",
      "            0.151]\":66,\"(0.151, 0.161]\":87,\"(0.161, 0.171]\":68,\"(0.171, 0.181]\":92,\"(0.181,\n",
      "            0.191]\":102,\"(0.191, 0.201]\":109,\"(0.201, 0.211]\":105,\"(0.211, 0.221]\":96,\"(0.221,\n",
      "            0.231]\":102,\"(0.231, 0.241]\":105,\"(0.241, 0.251]\":110,\"(0.251, 0.261]\":106,\"(0.261,\n",
      "            0.271]\":99,\"(0.271, 0.281]\":109,\"(0.281, 0.291]\":100,\"(0.291, 0.301]\":97,\"(0.301,\n",
      "            0.311]\":108,\"(0.311, 0.321]\":100,\"(0.321, 0.331]\":100,\"(0.331, 0.341]\":94,\"(0.341,\n",
      "            0.351]\":86,\"(0.351, 0.361]\":98,\"(0.361, 0.371]\":106,\"(0.371, 0.381]\":115,\"(0.381,\n",
      "            0.391]\":146,\"(0.391, 0.401]\":169,\"(0.401, 0.411]\":170,\"(0.411, 0.421]\":117,\"(0.421,\n",
      "            0.43]\":129,\"(0.43, 0.44]\":119,\"(0.44, 0.45]\":106,\"(0.45, 0.46]\":134,\"(0.46,\n",
      "            0.47]\":122,\"(0.47, 0.48]\":133,\"(0.48, 0.49]\":127,\"(0.49, 0.5]\":111,\"(0.5,\n",
      "            0.51]\":109,\"(0.51, 0.52]\":99,\"(0.52, 0.53]\":88,\"(0.53, 0.54]\":79,\"(0.54,\n",
      "            0.55]\":83,\"(0.55, 0.56]\":74,\"(0.56, 0.57]\":83,\"(0.57, 0.58]\":79,\"(0.58,\n",
      "            0.59]\":76,\"(0.59, 0.6]\":67,\"(0.6, 0.61]\":81,\"(0.61, 0.62]\":79,\"(0.62,\n",
      "            0.63]\":69,\"(0.63, 0.64]\":76,\"(0.64, 0.65]\":90,\"(0.65, 0.66]\":94,\"(0.66,\n",
      "            0.67]\":99,\"(0.67, 0.68]\":108,\"(0.68, 0.69]\":95,\"(0.69, 0.7]\":107,\"(0.7,\n",
      "            0.71]\":94,\"(0.71, 0.72]\":92,\"(0.72, 0.73]\":121,\"(0.73, 0.74]\":108,\"(0.74,\n",
      "            0.75]\":75,\"(0.75, 0.76]\":79,\"(0.76, 0.77]\":79,\"(0.77, 0.78]\":77,\"(0.78,\n",
      "            0.79]\":89,\"(0.79, 0.8]\":92,\"(0.8, 0.81]\":87,\"(0.81, 0.82]\":91,\"(0.82,\n",
      "            0.83]\":82,\"(0.83, 0.84]\":81,\"(0.84, 0.85]\":77,\"(0.85, 0.86]\":67,\"(0.86,\n",
      "            0.87]\":57,\"(0.87, 0.88]\":52,\"(0.88, 0.89]\":67,\"(0.89, 0.899]\":56,\"(0.899,\n",
      "            0.909]\":54,\"(0.909, 0.919]\":52,\"(0.919, 0.929]\":49,\"(0.929, 0.939]\":61,\"(0.939,\n",
      "            0.949]\":67,\"(0.949, 0.959]\":63,\"(0.959, 0.969]\":60,\"(0.969, 0.979]\":49,\"(0.979,\n",
      "            0.989]\":26,\"(0.989, 0.999]\":6}'\n",
      "      query_duration_sec: 0.04672431945800781\n",
      "    model:\n",
      "      cross_validation:\n",
      "        cv_duration_sec: 0.06284356117248535\n",
      "        scores:\n",
      "          explained-variance-score:\n",
      "            fold-1: -0.001176060252404687\n",
      "            fold-2: -0.010207618261960594\n",
      "            fold-3: -0.06841704012142014\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - &id002 !!python/object/apply:numpy.dtype\n",
      "              args:\n",
      "              - f8\n",
      "              - false\n",
      "              - true\n",
      "              state: !!python/tuple\n",
      "              - 3\n",
      "              - <\n",
      "              - null\n",
      "              - null\n",
      "              - null\n",
      "              - -1\n",
      "              - -1\n",
      "              - 0\n",
      "            - !!binary |\n",
      "              APyeFMFEU78=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              wJGm2xc9m78=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              wITPdceDsb8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              zmEM7FmDnj8=\n",
      "          explained-variance-score-SOME-TAG3:\n",
      "            fold-1: 0.057193087536291154\n",
      "            fold-2: -0.0839474479345057\n",
      "            fold-3: -0.02124117516171209\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              ICANkWlIrT8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              FU3H9OlhkL8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              ABx2d5R9tb8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              ljhhhQiQrT8=\n",
      "          explained-variance-score-SOME-TAG4:\n",
      "            fold-1: -0.05954520804110053\n",
      "            fold-2: 0.06353221141058452\n",
      "            fold-3: -0.11559290508112818\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              wEJAoqVDsD8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              NetC4SIMo78=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              oBv5In+Xvb8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              BNKT/Q4nsz8=\n",
      "          mean-absolute-error:\n",
      "            fold-1: 0.1781949694902849\n",
      "            fold-2: 0.20108865180405625\n",
      "            fold-3: 0.20000624112446902\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              +S2M30W9yT8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              nV+q22O3yD8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              AOEivxfPxj8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              HkMAK0+ZhT8=\n",
      "          mean-absolute-error-SOME-TAG3:\n",
      "            fold-1: 0.16744930157897417\n",
      "            fold-2: 0.19857001332638952\n",
      "            fold-3: 0.1985533518055695\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              EgCaAL5qyT8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              e4efmKMWyD8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              r4YCjfpuxT8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              VYzMznoJjj8=\n",
      "          mean-absolute-error-SOME-TAG4:\n",
      "            fold-1: 0.1889406374015956\n",
      "            fold-2: 0.20360729028172297\n",
      "            fold-3: 0.20145913044336855\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              4Ft+vs0Pyj8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              wze1HiRYyT8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              UDtD8TQvyD8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              MDDnZ359ej8=\n",
      "          mean-squared-error:\n",
      "            fold-1: 0.046781354618742416\n",
      "            fold-2: 0.056190475383456846\n",
      "            fold-3: 0.057109449823886486\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              p4GSJnM9rT8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              VLJezg5Sqz8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              7NZMyLnzpz8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              UTn5kOwdcz8=\n",
      "          mean-squared-error-SOME-TAG3:\n",
      "            fold-1: 0.04206239769303312\n",
      "            fold-2: 0.05659338570171074\n",
      "            fold-3: 0.055730936951266345\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              aHyE6c75rD8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              MG2MPEJZqj8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              skH33DOJpT8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              TiD/d1VDez8=\n",
      "          mean-squared-error-SOME-TAG4:\n",
      "            fold-1: 0.051500311544451706\n",
      "            fold-2: 0.05578756506520295\n",
      "            fold-3: 0.05848796269650663\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              13n7XSLyrT8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              ePcwYNtKrD8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              JWyisz9eqj8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              2inst72RZz8=\n",
      "          r2-score:\n",
      "            fold-1: -0.11897297365675708\n",
      "            fold-2: -0.050915676041659474\n",
      "            fold-3: -0.10055358538386527\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              0MLnlp4Rqr8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              A5ELmeYTt78=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              MLH2RgN1vr8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              N1HI3ZNunT8=\n",
      "          r2-score-SOME-TAG3:\n",
      "            fold-1: 0.025433976568473238\n",
      "            fold-2: -0.16317450882601992\n",
      "            fold-3: -0.02787256906561164\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              gHlGRl0Lmj8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              q+elLr9DrL8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              WGx5/ebixL8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              YdRWFb1StD8=\n",
      "          r2-score-SOME-TAG4:\n",
      "            fold-1: -0.2633799238819874\n",
      "            fold-2: 0.061343156742700966\n",
      "            fold-3: -0.1732346017021189\n",
      "            fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              wCsWyF5orz8=\n",
      "            fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              GBdizfYCwL8=\n",
      "            fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              MMDfdzfb0L8=\n",
      "            fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "            - *id002\n",
      "            - !!binary |\n",
      "              1xQI9uuEwT8=\n",
      "        splits:\n",
      "          fold-1-n-test: 2094\n",
      "          fold-1-n-train: 2094\n",
      "          fold-1-test-end: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1548845400000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-1-test-start: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1547589600000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-1-train-end: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1547589000000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-1-train-start: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1546333200000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-2-n-test: 2094\n",
      "          fold-2-n-train: 4188\n",
      "          fold-2-test-end: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1550142000000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-2-test-start: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1548846000000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-2-train-end: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1548845400000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-2-train-start: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1546333200000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-3-n-test: 2094\n",
      "          fold-3-n-train: 6282\n",
      "          fold-3-test-end: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1551398400000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-3-test-start: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1550142600000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-3-train-end: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1550142000000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "          fold-3-train-start: !!python/object/apply:pandas._libs.tslibs.timestamps._unpickle_timestamp\n",
      "          - 1546333200000000000\n",
      "          - null\n",
      "          - *id001\n",
      "          - 10\n",
      "      model_builder_version: 5.1.1\n",
      "      model_creation_date: '2023-06-07 08:13:04.494404+03:00'\n",
      "      model_meta:\n",
      "        aggregate-threshold: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id002\n",
      "        - !!binary |\n",
      "          3jAS82Ltyz8=\n",
      "        aggregate-thresholds-per-fold:\n",
      "          fold-0: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "          - *id002\n",
      "          - !!binary |\n",
      "            wVbC4Pdaxj8=\n",
      "          fold-1: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "          - *id002\n",
      "          - !!binary |\n",
      "            7jgu605lyj8=\n",
      "          fold-2: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "          - *id002\n",
      "          - !!binary |\n",
      "            3jAS82Ltyz8=\n",
      "        base_estimator: \"Pipeline(steps=[('step_0', PCA()),\\n                ('step_1',\\\n",
      "          \\ MultiOutputRegressor(estimator=LinearRegression()))])\"\n",
      "        feature-thresholds:\n",
      "        - 0.48542528577204486\n",
      "        - 0.494084774665874\n",
      "        feature-thresholds-per-fold:\n",
      "          SOME-TAG3:\n",
      "            fold-0: 0.44230855404164027\n",
      "            fold-1: 0.5146225991588878\n",
      "            fold-2: 0.48542528577204486\n",
      "          SOME-TAG4:\n",
      "            fold-0: 0.5394854306937662\n",
      "            fold-1: 0.5331993905947455\n",
      "            fold-2: 0.494084774665874\n",
      "        scaler: MinMaxScaler()\n",
      "        shuffle: false\n",
      "        smooth-aggregate-thresholds-per-fold: {}\n",
      "        smooth-feature-thresholds-per-fold: {}\n",
      "        smoothing-method: null\n",
      "        window: null\n",
      "      model_offset: 0\n",
      "      model_training_duration_sec: 0.005138874053955078\n",
      "  user_defined:\n",
      "    global-metadata: {}\n",
      "    machine-metadata:\n",
      "      information: Some sweet information about the model\n",
      "model:\n",
      "  gordo.machine.model.anomaly.diff.DiffBasedAnomalyDetector:\n",
      "    base_estimator:\n",
      "      sklearn.pipeline.Pipeline:\n",
      "        steps:\n",
      "        - sklearn.decomposition.PCA\n",
      "        - sklearn.multioutput.MultiOutputRegressor:\n",
      "            estimator: sklearn.linear_model.LinearRegression\n",
      "name: crazy-sweet-name\n",
      "project_name: local-build\n",
      "runtime:\n",
      "  builder:\n",
      "    image: gordo-base\n",
      "    remote_logging:\n",
      "      enable: false\n",
      "    resources:\n",
      "      limits:\n",
      "        cpu: 1001\n",
      "        memory: 31200\n",
      "      requests:\n",
      "        cpu: 1001\n",
      "        memory: 3900\n",
      "  client:\n",
      "    image: gordo-base\n",
      "    max_instances: 30\n",
      "    resources:\n",
      "      limits:\n",
      "        cpu: 2000\n",
      "        memory: 4000\n",
      "      requests:\n",
      "        cpu: 100\n",
      "        memory: 3500\n",
      "  deployer:\n",
      "    image: gordo-base\n",
      "  influx:\n",
      "    enable: true\n",
      "    resources:\n",
      "      limits:\n",
      "        cpu: 10020\n",
      "        memory: 3220\n",
      "      requests:\n",
      "        cpu: 510\n",
      "        memory: 3220\n",
      "  prometheus_metrics_server:\n",
      "    image: gordo-base\n",
      "    resources:\n",
      "      limits:\n",
      "        cpu: 200\n",
      "        memory: 1000\n",
      "      requests:\n",
      "        cpu: 100\n",
      "        memory: 200\n",
      "  reporters: []\n",
      "  server:\n",
      "    image: gordo-base\n",
      "    resources:\n",
      "      limits:\n",
      "        cpu: 2000\n",
      "        memory: 6000\n",
      "      requests:\n",
      "        cpu: 1000\n",
      "        memory: 3000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(yaml.dump(metadata.to_dict()))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}