run_keras_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/sergeymiller1996/anaconda3/lib/python3.5/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import scipy.stats as sps\n",
    "import pandas as pd\n",
    "import h5py\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "\n",
    "from copy import deepcopy\n",
    "\n",
    "from numpy import array\n",
    "from os import environ\n",
    "from os.path import join\n",
    "from sys import argv\n",
    "\n",
    "from glob import glob\n",
    "from numpy import zeros\n",
    "from os.path import basename, join\n",
    "\n",
    "from keras.models import load_model\n",
    "from os import environ\n",
    "from os.path import abspath, dirname, join\n",
    "\n",
    "from copy import copy\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.models import model_from_json\n",
    "\n",
    "from keras.layers import InputLayer, Input, Dense, Activation, Dropout, BatchNormalization, Reshape\n",
    "from keras.layers import Average, Conv2D, MaxPooling2D, Flatten\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.activations import relu, softmax, sigmoid\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.regularizers import l2\n",
    "\n",
    "from keras.constraints import maxnorm\n",
    "\n",
    "from keras.optimizers import SGD, Adam\n",
    "from keras.utils import np_utils\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "from keras.losses import binary_crossentropy, mse\n",
    "\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "\n",
    "import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datagen_Miller\n",
    "import models.keras_lstm\n",
    "import models.dnn\n",
    "import feature_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from importlib import reload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def score(y_true, y_pred):\n",
    "    return 1 / (keras.losses.mse(y_true, y_pred)) ** 0.5\n",
    "\n",
    "def run_model(X_train, y_train, X_val, y_val, \n",
    "          bad_columns=None,\n",
    "          batch_size=1000,\n",
    "          feature_expander=None,\n",
    "          forecast_win=10,\n",
    "          features_win=100,\n",
    "          epochs=50,\n",
    "          input_shape=None,\n",
    "          path_to_load=None, \n",
    "          path_to_save=None,\n",
    "          isLstm=False,\n",
    "          epoch_train=100,\n",
    "          epoch_val=100,\n",
    "         ):\n",
    "    \n",
    "    assert isinstance(X_train, pd.DataFrame)\n",
    "    assert input_shape is not None\n",
    "    \n",
    "    if path_to_load is None:\n",
    "        if isLstm:\n",
    "            model = models.keras_lstm.reccurent_model(\n",
    "                                              input_shape=input_shape)\n",
    "        else:\n",
    "            model = models.dnn.nn_1(input_shape=input_shape)\n",
    "    else:\n",
    "        model = load_model(path_to_load)\n",
    "        optimizer = Adam(1e-3, clipvalue=0.5)\n",
    "        model.compile(loss='mean_squared_error', \n",
    "                      optimizer=optimizer, metrics=[score])\n",
    "    \n",
    "    datagen = datagen_Miller.DataGenerator(\n",
    "                                        batch_size,\n",
    "                                        feature_expander,\n",
    "                                        forecast_win, \n",
    "                                        features_win,\n",
    "                                        isLstm,\n",
    "                                          )\n",
    "    datagen.fit()\n",
    "    \n",
    "    filepath = \"weights.{epoch:03d}-{val_loss:.3f}.hdf5\"\n",
    "    \n",
    "    print('val steps {}'.format(X_val.shape[0] // batch_size))\n",
    "    \n",
    "    model.fit_generator(datagen.flow(X_train, y_train),\n",
    "                        steps_per_epoch=min(epoch_train, X_train.shape[0] / batch_size), \n",
    "                        epochs=epochs, \n",
    "                        shuffle=False,\n",
    "        callbacks=[\n",
    "            datagen_Miller.TelegramCallback(),\n",
    "            keras.callbacks.ModelCheckpoint(filepath, \n",
    "                                        monitor='val_loss', \n",
    "                                        verbose=0, \n",
    "                                        save_best_only=False, \n",
    "                                        save_weights_only=True, \n",
    "                                        mode='auto', \n",
    "                                        period=1)\n",
    "                ],\n",
    "        validation_data=datagen.flow(X_val, y_val),\n",
    "        validation_steps=min(epoch_val, X_val.shape[0] // batch_size),\n",
    "                       )\n",
    "    \n",
    "    if path_to_save:\n",
    "        model.save_weights(path_to_save)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'feature_generator' from '/home/sergeymiller1996/docs/alpha/NightPy/feature_generator.py'>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(datagen_Miller)\n",
    "reload(models.keras_lstm)\n",
    "reload(models.dnn)\n",
    "reload(feature_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 24 s, sys: 2.15 s, total: 26.2 s\n",
      "Wall time: 27.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# X_train_tea = pd.read_csv('splits/tea_full_train.csv')\n",
    "# X_val_tea = pd.read_csv('splits/tea_full_validation.csv')\n",
    "# X_train_coffee = pd.read_csv('splits/coffee_full_train.csv')\n",
    "# X_val_coffee = pd.read_csv('splits/coffee_full_validation.csv')\n",
    "X_test_tea = pd.read_csv('splits/tea_full_test.csv')\n",
    "X_test_coffee = pd.read_csv('splits/coffee_full_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filter_columns = ['Unnamed: 0', '0_ID', 'Y', '1_TIME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_test = np.array(X_test_tea['Y'])\n",
    "X_test_tea = pd.DataFrame(X_test_tea.drop(filter_columns, axis=1), dtype=np.float32)\n",
    "X_test_coffee = pd.DataFrame(X_test_coffee.drop(filter_columns, axis=1), dtype=np.float32)\n",
    "# y_val = np.array(X_val_tea['Y'])\n",
    "# X_val_tea = pd.DataFrame(X_val_tea.drop(filter_columns, axis=1), dtype=np.float32)\n",
    "# X_val_coffee = pd.DataFrame(X_val_coffee.drop(filter_columns, axis=1), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3134616"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_tea.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final = pd.concat((X_test_tea, X_test_coffee), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sz = int(0.95 * final.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_train, final_test = final.iloc[:sz], final.iloc[sz:]\n",
    "y_train, y_test = y_test[:sz], y_test[sz:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "val steps 783\n",
      "Epoch 1/30\n",
      "100/100 [==============================] - 79s 786ms/step - loss: 7.1952 - dense_23_loss: 0.0393 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 469.6187 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 38.6640 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 12.1165 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 2.1093 - val_dense_23_loss: 0.0390 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 207.9584 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 39.8369 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 10.6272 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 2/30\n",
      "100/100 [==============================] - 74s 740ms/step - loss: 0.9215 - dense_23_loss: 0.0251 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 513.2111 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 30.1396 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 5.8170 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.3060 - val_dense_23_loss: 0.0280 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 477.7090 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 19.9148 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 1.1365 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 3/30\n",
      "100/100 [==============================] - 73s 733ms/step - loss: 0.1761 - dense_23_loss: 0.0505 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 360.6746 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 24.3305 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 2.3360 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.1156 - val_dense_23_loss: 0.0647 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 144.1230 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 17.6339 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1604 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 4/30\n",
      "100/100 [==============================] - 75s 750ms/step - loss: 0.0564 - dense_23_loss: 0.0321 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 657.4731 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 23.9763 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 1.5049 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0428 - val_dense_23_loss: 0.0304 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 2389.2787 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 18.6294 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.2481 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 5/30\n",
      "100/100 [==============================] - 75s 748ms/step - loss: 0.0365 - dense_23_loss: 0.0275 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 361.9115 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 21.5032 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.3972 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0417 - val_dense_23_loss: 0.0337 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 467.6317 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 17.5889 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1389 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 6/30\n",
      "100/100 [==============================] - 75s 750ms/step - loss: 0.0349 - dense_23_loss: 0.0285 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 730.1597 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 19.5891 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.2401 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0337 - val_dense_23_loss: 0.0275 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 416.9774 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 19.1870 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.2301 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 7/30\n",
      "100/100 [==============================] - 74s 736ms/step - loss: 0.0386 - dense_23_loss: 0.0330 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 370.6358 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 18.7542 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.2040 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0314 - val_dense_23_loss: 0.0262 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 316.1927 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 19.2049 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.2438 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 8/30\n",
      "100/100 [==============================] - 75s 749ms/step - loss: 0.0345 - dense_23_loss: 0.0291 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 477.7673 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 17.7765 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1758 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0340 - val_dense_23_loss: 0.0286 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 454.8533 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 17.1196 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1551 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 9/30\n",
      "100/100 [==============================] - 74s 736ms/step - loss: 0.0344 - dense_23_loss: 0.0293 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 476.5781 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 16.8808 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1499 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0326 - val_dense_23_loss: 0.0281 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 482.5686 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 16.2516 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1209 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 10/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 [==============================] - 74s 739ms/step - loss: 0.0314 - dense_23_loss: 0.0269 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 522.3518 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 16.5568 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1397 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0353 - val_dense_23_loss: 0.0303 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 415.6011 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 16.2698 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1380 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 11/30\n",
      "100/100 [==============================] - 74s 738ms/step - loss: 0.0359 - dense_23_loss: 0.0316 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 391.5521 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 16.2461 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1329 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0304 - val_dense_23_loss: 0.0270 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 273.6310 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 16.1674 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1202 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 12/30\n",
      "100/100 [==============================] - 73s 730ms/step - loss: 0.0323 - dense_23_loss: 0.0281 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 447.4841 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 16.2767 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1315 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0304 - val_dense_23_loss: 0.0264 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 305.5998 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 16.3881 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1532 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 13/30\n",
      "100/100 [==============================] - 73s 732ms/step - loss: 0.0319 - dense_23_loss: 0.0275 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 411.0300 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 16.0293 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1267 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0287 - val_dense_23_loss: 0.0249 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 743.6216 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.6757 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1034 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 14/30\n",
      "100/100 [==============================] - 75s 745ms/step - loss: 0.0338 - dense_23_loss: 0.0295 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 706.5712 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.8288 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1241 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0333 - val_dense_23_loss: 0.0296 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 519.0106 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.6934 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1111 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 15/30\n",
      "100/100 [==============================] - 75s 751ms/step - loss: 0.0335 - dense_23_loss: 0.0297 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 515.7265 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.7379 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1223 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0327 - val_dense_23_loss: 0.0288 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 998.9364 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.7215 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1251 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 16/30\n",
      "100/100 [==============================] - 73s 732ms/step - loss: 0.0314 - dense_23_loss: 0.0278 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 562.4118 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.6485 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1189 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0317 - val_dense_23_loss: 0.0287 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 292.0680 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.6010 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1125 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 17/30\n",
      "100/100 [==============================] - 75s 747ms/step - loss: 0.0332 - dense_23_loss: 0.0298 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 566.4124 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.5834 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1166 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0306 - val_dense_23_loss: 0.0280 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 325.2688 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.6009 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1161 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 18/30\n",
      "100/100 [==============================] - 73s 730ms/step - loss: 0.0290 - dense_23_loss: 0.0258 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 327.7095 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.5269 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1152 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0304 - val_dense_23_loss: 0.0270 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 408.2605 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.4332 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1065 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 19/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100/100 [==============================] - 75s 755ms/step - loss: 0.0352 - dense_23_loss: 0.0315 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 17198.0820 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.4119 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1127 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0303 - val_dense_23_loss: 0.0270 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 975.8430 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.5464 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1257 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 20/30\n",
      "100/100 [==============================] - 74s 743ms/step - loss: 0.0344 - dense_23_loss: 0.0314 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 366.4349 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.4069 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1118 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0308 - val_dense_23_loss: 0.0277 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 383.9674 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.5020 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1253 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 21/30\n",
      "100/100 [==============================] - 74s 737ms/step - loss: 0.0360 - dense_23_loss: 0.0330 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 281.4711 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.3588 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1103 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0281 - val_dense_23_loss: 0.0252 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 350.0565 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.4475 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1205 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 22/30\n",
      "100/100 [==============================] - 74s 740ms/step - loss: 0.0324 - dense_23_loss: 0.0291 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 859.8903 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.2974 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1082 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0299 - val_dense_23_loss: 0.0268 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 329.3904 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.1671 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.0973 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 23/30\n",
      "100/100 [==============================] - 74s 739ms/step - loss: 0.0349 - dense_23_loss: 0.0318 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 1043.7785 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.2363 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1064 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0292 - val_dense_23_loss: 0.0259 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 372.2097 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.3147 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1217 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 24/30\n",
      "100/100 [==============================] - 74s 741ms/step - loss: 0.0307 - dense_23_loss: 0.0279 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 451.6356 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.1949 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1044 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0297 - val_dense_23_loss: 0.0269 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 308.9820 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.1541 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1015 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 25/30\n",
      "100/100 [==============================] - 74s 742ms/step - loss: 0.0259 - dense_23_loss: 0.0232 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 320.5038 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.1529 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1036 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0291 - val_dense_23_loss: 0.0263 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 304.4619 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.2434 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.1193 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 26/30\n",
      "100/100 [==============================] - 75s 746ms/step - loss: 0.0319 - dense_23_loss: 0.0293 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 418.9056 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.1290 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1025 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0300 - val_dense_23_loss: 0.0272 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 2361.0907 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.0854 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.0984 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 27/30\n",
      "100/100 [==============================] - 73s 734ms/step - loss: 0.0316 - dense_23_loss: 0.0290 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 486.6495 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.1170 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1015 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00 - val_loss: 0.0300 - val_dense_23_loss: 0.0271 - val_lstm_13_loss_1: 0.0000e+00 - val_lstm_13_loss_2: 0.0000e+00 - val_dense_23_score: 2391.3159 - val_dense_23_fake_loss_1: 0.0000e+00 - val_dense_23_fake_loss_2: 0.0000e+00 - val_lstm_13_score_1: 15.0745 - val_lstm_13_fake_loss_1: 0.0000e+00 - val_lstm_13_fake_loss_2: 0.0000e+00 - val_lstm_13_score_2: 0.0986 - val_lstm_13_fake_loss_3: 0.0000e+00 - val_lstm_13_fake_loss_4: 0.0000e+00\n",
      "Epoch 28/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 99/100 [============================>.] - ETA: 0s - loss: 0.0284 - dense_23_loss: 0.0256 - lstm_13_loss_1: 0.0000e+00 - lstm_13_loss_2: 0.0000e+00 - dense_23_score: 618.0022 - dense_23_fake_loss_1: 0.0000e+00 - dense_23_fake_loss_2: 0.0000e+00 - lstm_13_score_1: 15.0617 - lstm_13_fake_loss_1: 0.0000e+00 - lstm_13_fake_loss_2: 0.0000e+00 - lstm_13_score_2: 0.1002 - lstm_13_fake_loss_3: 0.0000e+00 - lstm_13_fake_loss_4: 0.0000e+00"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "try:\n",
    "    run_model(final_train, y_train, \n",
    "              final_test, y_test, batch_size=200, epoch_train=100, epoch_val=400,\n",
    "          feature_expander=None,\n",
    "          input_shape=80,\n",
    "          features_win=2,forecast_win=1, epochs=30, path_to_save='init_model', isLstm=True)\n",
    "except KeyboardInterrupt:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = load_model(path_to_load)\n",
    "        optimizer = Adam(1e-3, clipvalue=0.5)\n",
    "        model.compile(loss='mean_squared_error', \n",
    "                      optimizer=optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final = pd.concat((X_test_tea, X_test_coffee), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "try:\n",
    "    run_model(, y_train, \n",
    "              pd.concat((X_val_tea, X_val_coffee), axis=1), y_val, batch_size=1000, epoch_len=10,\n",
    "          feature_expander=feature_generator.count_all_features,\n",
    "          input_shape=262,\n",
    "          features_win=2,forecast_win=1, epochs=10, path_to_save='init_model', isLstm=True)\n",
    "except KeyboardInterrupt:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>0_ID</th>\n",
       "      <th>1_TIME</th>\n",
       "      <th>BID_P_1</th>\n",
       "      <th>BID_V_1</th>\n",
       "      <th>BID_P_2</th>\n",
       "      <th>BID_V_2</th>\n",
       "      <th>BID_P_3</th>\n",
       "      <th>BID_V_3</th>\n",
       "      <th>BID_P_4</th>\n",
       "      <th>...</th>\n",
       "      <th>ASK_P_6</th>\n",
       "      <th>ASK_V_6</th>\n",
       "      <th>ASK_P_7</th>\n",
       "      <th>ASK_V_7</th>\n",
       "      <th>ASK_P_8</th>\n",
       "      <th>ASK_V_8</th>\n",
       "      <th>ASK_P_9</th>\n",
       "      <th>ASK_V_9</th>\n",
       "      <th>ASK_P_10</th>\n",
       "      <th>ASK_V_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5617875</td>\n",
       "      <td>TEA</td>\n",
       "      <td>1970.01.02 17:59:17.691677</td>\n",
       "      <td>2561</td>\n",
       "      <td>81</td>\n",
       "      <td>2560</td>\n",
       "      <td>73</td>\n",
       "      <td>2559</td>\n",
       "      <td>99</td>\n",
       "      <td>2558</td>\n",
       "      <td>...</td>\n",
       "      <td>2567</td>\n",
       "      <td>109.0</td>\n",
       "      <td>2568.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>2569.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>2570.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>2571.0</td>\n",
       "      <td>124.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5617876</td>\n",
       "      <td>TEA</td>\n",
       "      <td>1970.01.02 17:59:17.692349</td>\n",
       "      <td>2561</td>\n",
       "      <td>81</td>\n",
       "      <td>2560</td>\n",
       "      <td>73</td>\n",
       "      <td>2559</td>\n",
       "      <td>99</td>\n",
       "      <td>2558</td>\n",
       "      <td>...</td>\n",
       "      <td>2567</td>\n",
       "      <td>109.0</td>\n",
       "      <td>2568.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>2569.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>2570.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>2571.0</td>\n",
       "      <td>124.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5617877</td>\n",
       "      <td>TEA</td>\n",
       "      <td>1970.01.02 17:59:17.692369</td>\n",
       "      <td>2561</td>\n",
       "      <td>81</td>\n",
       "      <td>2560</td>\n",
       "      <td>73</td>\n",
       "      <td>2559</td>\n",
       "      <td>99</td>\n",
       "      <td>2558</td>\n",
       "      <td>...</td>\n",
       "      <td>2567</td>\n",
       "      <td>109.0</td>\n",
       "      <td>2568.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>2569.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>2570.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>2571.0</td>\n",
       "      <td>124.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5617878</td>\n",
       "      <td>TEA</td>\n",
       "      <td>1970.01.02 17:59:17.692587</td>\n",
       "      <td>2561</td>\n",
       "      <td>81</td>\n",
       "      <td>2560</td>\n",
       "      <td>73</td>\n",
       "      <td>2559</td>\n",
       "      <td>99</td>\n",
       "      <td>2558</td>\n",
       "      <td>...</td>\n",
       "      <td>2567</td>\n",
       "      <td>109.0</td>\n",
       "      <td>2568.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>2569.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>2570.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>2571.0</td>\n",
       "      <td>124.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5617879</td>\n",
       "      <td>TEA</td>\n",
       "      <td>1970.01.02 17:59:17.692607</td>\n",
       "      <td>2561</td>\n",
       "      <td>81</td>\n",
       "      <td>2560</td>\n",
       "      <td>73</td>\n",
       "      <td>2559</td>\n",
       "      <td>99</td>\n",
       "      <td>2558</td>\n",
       "      <td>...</td>\n",
       "      <td>2567</td>\n",
       "      <td>109.0</td>\n",
       "      <td>2568.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>2569.0</td>\n",
       "      <td>104.0</td>\n",
       "      <td>2570.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>2571.0</td>\n",
       "      <td>124.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0 0_ID                      1_TIME  BID_P_1  BID_V_1  BID_P_2  \\\n",
       "0     5617875  TEA  1970.01.02 17:59:17.691677     2561       81     2560   \n",
       "1     5617876  TEA  1970.01.02 17:59:17.692349     2561       81     2560   \n",
       "2     5617877  TEA  1970.01.02 17:59:17.692369     2561       81     2560   \n",
       "3     5617878  TEA  1970.01.02 17:59:17.692587     2561       81     2560   \n",
       "4     5617879  TEA  1970.01.02 17:59:17.692607     2561       81     2560   \n",
       "\n",
       "   BID_V_2  BID_P_3  BID_V_3  BID_P_4    ...     ASK_P_6  ASK_V_6  ASK_P_7  \\\n",
       "0       73     2559       99     2558    ...        2567    109.0   2568.0   \n",
       "1       73     2559       99     2558    ...        2567    109.0   2568.0   \n",
       "2       73     2559       99     2558    ...        2567    109.0   2568.0   \n",
       "3       73     2559       99     2558    ...        2567    109.0   2568.0   \n",
       "4       73     2559       99     2558    ...        2567    109.0   2568.0   \n",
       "\n",
       "   ASK_V_7  ASK_P_8  ASK_V_8  ASK_P_9  ASK_V_9  ASK_P_10  ASK_V_10  \n",
       "0     92.0   2569.0    105.0   2570.0     77.0    2571.0     124.0  \n",
       "1     92.0   2569.0    105.0   2570.0     77.0    2571.0     124.0  \n",
       "2     92.0   2569.0    105.0   2570.0     77.0    2571.0     124.0  \n",
       "3     92.0   2569.0    105.0   2570.0     77.0    2571.0     124.0  \n",
       "4     92.0   2569.0    104.0   2570.0     77.0    2571.0     124.0  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.9'"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}