Ray_tune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/drucev/opt/anaconda3/envs/test4/lib/python3.8/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "non-resource variables are not supported in the long term\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import multiprocessing\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# import tensorflow as tf\n",
    "\n",
    "import ray\n",
    "from ray import tune\n",
    "from ray.rllib.agents.ppo import PPOTrainer, DEFAULT_CONFIG\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import gym\n",
    "\n",
    "n_cpus = multiprocessing.cpu_count()\n",
    "n_cpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# n_gpus = len(tf.config.list_physical_devices('GPU'))\n",
    "n_gpus = 0\n",
    "n_gpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-11-21 11:36:13,848\tINFO services.py:1090 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://192.168.0.23:8265\u001b[39m\u001b[22m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'node_ip_address': '192.168.0.23',\n",
       " 'raylet_ip_address': '192.168.0.23',\n",
       " 'redis_address': '192.168.0.23:6379',\n",
       " 'object_store_address': '/tmp/ray/session_2020-11-21_11-36-13_372673_20570/sockets/plasma_store',\n",
       " 'raylet_socket_name': '/tmp/ray/session_2020-11-21_11-36-13_372673_20570/sockets/raylet',\n",
       " 'webui_url': '192.168.0.23:8265',\n",
       " 'session_dir': '/tmp/ray/session_2020-11-21_11-36-13_372673_20570',\n",
       " 'metrics_export_port': 62521,\n",
       " 'node_id': '73b921d916afb2d08b1816771ddab3069bbb1642'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# initialize ray\n",
    "# https://ray.readthedocs.io/en/latest/package-ref.html#ray.init\n",
    "ray.init(ignore_reinit_error=True, log_to_driver=False, dashboard_host='0.0.0.0')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-11-21 11:17:52,985\tINFO trainer.py:588 -- Executing eagerly, with eager_tracing=False\n",
      "2020-11-21 11:17:52,986\tINFO trainer.py:1064 -- `_use_trajectory_view_api` only supported for PyTorch so far! Will run w/o.\n",
      "2020-11-21 11:17:52,986\tINFO trainer.py:617 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/drucev/opt/anaconda3/envs/test4/lib/python3.8/site-packages/ray/rllib/policy/tf_policy.py:875: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Prefer Variable.assign which has equivalent behavior in 2.X.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'episode_reward_max': 121.0,\n",
       " 'episode_reward_min': 8.0,\n",
       " 'episode_reward_mean': 20.984251968503937,\n",
       " 'episode_len_mean': 20.984251968503937,\n",
       " 'episodes_this_iter': 381,\n",
       " 'policy_reward_min': {},\n",
       " 'policy_reward_max': {},\n",
       " 'policy_reward_mean': {},\n",
       " 'custom_metrics': {},\n",
       " 'hist_stats': {'episode_reward': [14.0,\n",
       "   15.0,\n",
       "   14.0,\n",
       "   22.0,\n",
       "   11.0,\n",
       "   9.0,\n",
       "   29.0,\n",
       "   33.0,\n",
       "   18.0,\n",
       "   16.0,\n",
       "   16.0,\n",
       "   34.0,\n",
       "   39.0,\n",
       "   28.0,\n",
       "   12.0,\n",
       "   14.0,\n",
       "   24.0,\n",
       "   35.0,\n",
       "   13.0,\n",
       "   16.0,\n",
       "   12.0,\n",
       "   25.0,\n",
       "   26.0,\n",
       "   20.0,\n",
       "   13.0,\n",
       "   11.0,\n",
       "   10.0,\n",
       "   25.0,\n",
       "   22.0,\n",
       "   32.0,\n",
       "   19.0,\n",
       "   18.0,\n",
       "   15.0,\n",
       "   9.0,\n",
       "   27.0,\n",
       "   10.0,\n",
       "   20.0,\n",
       "   15.0,\n",
       "   19.0,\n",
       "   29.0,\n",
       "   14.0,\n",
       "   14.0,\n",
       "   13.0,\n",
       "   14.0,\n",
       "   11.0,\n",
       "   15.0,\n",
       "   14.0,\n",
       "   16.0,\n",
       "   46.0,\n",
       "   14.0,\n",
       "   15.0,\n",
       "   18.0,\n",
       "   14.0,\n",
       "   30.0,\n",
       "   22.0,\n",
       "   15.0,\n",
       "   17.0,\n",
       "   22.0,\n",
       "   13.0,\n",
       "   13.0,\n",
       "   33.0,\n",
       "   14.0,\n",
       "   12.0,\n",
       "   18.0,\n",
       "   15.0,\n",
       "   20.0,\n",
       "   41.0,\n",
       "   12.0,\n",
       "   16.0,\n",
       "   33.0,\n",
       "   24.0,\n",
       "   13.0,\n",
       "   18.0,\n",
       "   14.0,\n",
       "   11.0,\n",
       "   15.0,\n",
       "   32.0,\n",
       "   20.0,\n",
       "   16.0,\n",
       "   27.0,\n",
       "   21.0,\n",
       "   12.0,\n",
       "   25.0,\n",
       "   31.0,\n",
       "   11.0,\n",
       "   21.0,\n",
       "   13.0,\n",
       "   12.0,\n",
       "   21.0,\n",
       "   19.0,\n",
       "   21.0,\n",
       "   54.0,\n",
       "   12.0,\n",
       "   18.0,\n",
       "   14.0,\n",
       "   44.0,\n",
       "   66.0,\n",
       "   37.0,\n",
       "   16.0,\n",
       "   17.0,\n",
       "   13.0,\n",
       "   38.0,\n",
       "   33.0,\n",
       "   17.0,\n",
       "   14.0,\n",
       "   24.0,\n",
       "   33.0,\n",
       "   26.0,\n",
       "   25.0,\n",
       "   43.0,\n",
       "   25.0,\n",
       "   19.0,\n",
       "   14.0,\n",
       "   20.0,\n",
       "   11.0,\n",
       "   72.0,\n",
       "   16.0,\n",
       "   37.0,\n",
       "   11.0,\n",
       "   12.0,\n",
       "   16.0,\n",
       "   12.0,\n",
       "   20.0,\n",
       "   19.0,\n",
       "   25.0,\n",
       "   10.0,\n",
       "   41.0,\n",
       "   10.0,\n",
       "   29.0,\n",
       "   39.0,\n",
       "   10.0,\n",
       "   17.0,\n",
       "   11.0,\n",
       "   19.0,\n",
       "   24.0,\n",
       "   16.0,\n",
       "   18.0,\n",
       "   13.0,\n",
       "   24.0,\n",
       "   18.0,\n",
       "   14.0,\n",
       "   21.0,\n",
       "   22.0,\n",
       "   13.0,\n",
       "   14.0,\n",
       "   13.0,\n",
       "   15.0,\n",
       "   18.0,\n",
       "   12.0,\n",
       "   52.0,\n",
       "   17.0,\n",
       "   17.0,\n",
       "   14.0,\n",
       "   11.0,\n",
       "   27.0,\n",
       "   60.0,\n",
       "   23.0,\n",
       "   11.0,\n",
       "   15.0,\n",
       "   25.0,\n",
       "   21.0,\n",
       "   56.0,\n",
       "   17.0,\n",
       "   26.0,\n",
       "   15.0,\n",
       "   26.0,\n",
       "   21.0,\n",
       "   15.0,\n",
       "   13.0,\n",
       "   12.0,\n",
       "   11.0,\n",
       "   33.0,\n",
       "   24.0,\n",
       "   15.0,\n",
       "   17.0,\n",
       "   12.0,\n",
       "   17.0,\n",
       "   14.0,\n",
       "   15.0,\n",
       "   20.0,\n",
       "   9.0,\n",
       "   14.0,\n",
       "   15.0,\n",
       "   11.0,\n",
       "   15.0,\n",
       "   17.0,\n",
       "   22.0,\n",
       "   13.0,\n",
       "   26.0,\n",
       "   15.0,\n",
       "   19.0,\n",
       "   23.0,\n",
       "   27.0,\n",
       "   12.0,\n",
       "   22.0,\n",
       "   30.0,\n",
       "   18.0,\n",
       "   52.0,\n",
       "   48.0,\n",
       "   32.0,\n",
       "   20.0,\n",
       "   13.0,\n",
       "   20.0,\n",
       "   16.0,\n",
       "   20.0,\n",
       "   31.0,\n",
       "   13.0,\n",
       "   26.0,\n",
       "   37.0,\n",
       "   12.0,\n",
       "   11.0,\n",
       "   24.0,\n",
       "   10.0,\n",
       "   45.0,\n",
       "   17.0,\n",
       "   13.0,\n",
       "   12.0,\n",
       "   15.0,\n",
       "   17.0,\n",
       "   16.0,\n",
       "   35.0,\n",
       "   26.0,\n",
       "   11.0,\n",
       "   27.0,\n",
       "   16.0,\n",
       "   10.0,\n",
       "   10.0,\n",
       "   18.0,\n",
       "   28.0,\n",
       "   22.0,\n",
       "   36.0,\n",
       "   12.0,\n",
       "   13.0,\n",
       "   23.0,\n",
       "   11.0,\n",
       "   26.0,\n",
       "   22.0,\n",
       "   24.0,\n",
       "   17.0,\n",
       "   25.0,\n",
       "   13.0,\n",
       "   46.0,\n",
       "   29.0,\n",
       "   24.0,\n",
       "   21.0,\n",
       "   15.0,\n",
       "   14.0,\n",
       "   28.0,\n",
       "   8.0,\n",
       "   17.0,\n",
       "   27.0,\n",
       "   31.0,\n",
       "   40.0,\n",
       "   26.0,\n",
       "   24.0,\n",
       "   32.0,\n",
       "   60.0,\n",
       "   11.0,\n",
       "   15.0,\n",
       "   15.0,\n",
       "   14.0,\n",
       "   9.0,\n",
       "   18.0,\n",
       "   26.0,\n",
       "   15.0,\n",
       "   24.0,\n",
       "   32.0,\n",
       "   22.0,\n",
       "   13.0,\n",
       "   22.0,\n",
       "   18.0,\n",
       "   14.0,\n",
       "   20.0,\n",
       "   12.0,\n",
       "   11.0,\n",
       "   38.0,\n",
       "   17.0,\n",
       "   18.0,\n",
       "   28.0,\n",
       "   22.0,\n",
       "   20.0,\n",
       "   15.0,\n",
       "   26.0,\n",
       "   33.0,\n",
       "   121.0,\n",
       "   12.0,\n",
       "   20.0,\n",
       "   20.0,\n",
       "   20.0,\n",
       "   23.0,\n",
       "   36.0,\n",
       "   8.0,\n",
       "   12.0,\n",
       "   42.0,\n",
       "   17.0,\n",
       "   13.0,\n",
       "   19.0,\n",
       "   35.0,\n",
       "   17.0,\n",
       "   19.0,\n",
       "   42.0,\n",
       "   21.0,\n",
       "   28.0,\n",
       "   12.0,\n",
       "   14.0,\n",
       "   13.0,\n",
       "   14.0,\n",
       "   19.0,\n",
       "   30.0,\n",
       "   11.0,\n",
       "   15.0,\n",
       "   17.0,\n",
       "   29.0,\n",
       "   25.0,\n",
       "   15.0,\n",
       "   12.0,\n",
       "   20.0,\n",
       "   82.0,\n",
       "   18.0,\n",
       "   12.0,\n",
       "   44.0,\n",
       "   12.0,\n",
       "   22.0,\n",
       "   11.0,\n",
       "   23.0,\n",
       "   14.0,\n",
       "   29.0,\n",
       "   18.0,\n",
       "   42.0,\n",
       "   14.0,\n",
       "   50.0,\n",
       "   14.0,\n",
       "   22.0,\n",
       "   20.0,\n",
       "   20.0,\n",
       "   9.0,\n",
       "   25.0,\n",
       "   12.0,\n",
       "   14.0,\n",
       "   9.0,\n",
       "   8.0,\n",
       "   18.0,\n",
       "   17.0,\n",
       "   23.0,\n",
       "   23.0,\n",
       "   15.0,\n",
       "   11.0,\n",
       "   21.0,\n",
       "   14.0,\n",
       "   13.0,\n",
       "   9.0,\n",
       "   17.0,\n",
       "   21.0,\n",
       "   20.0,\n",
       "   11.0,\n",
       "   14.0,\n",
       "   13.0,\n",
       "   40.0,\n",
       "   17.0,\n",
       "   23.0,\n",
       "   17.0,\n",
       "   10.0,\n",
       "   11.0,\n",
       "   11.0,\n",
       "   28.0,\n",
       "   16.0,\n",
       "   12.0,\n",
       "   14.0,\n",
       "   12.0,\n",
       "   16.0,\n",
       "   14.0,\n",
       "   21.0,\n",
       "   16.0,\n",
       "   35.0,\n",
       "   17.0,\n",
       "   17.0,\n",
       "   20.0,\n",
       "   15.0,\n",
       "   20.0,\n",
       "   49.0,\n",
       "   10.0],\n",
       "  'episode_lengths': [14,\n",
       "   15,\n",
       "   14,\n",
       "   22,\n",
       "   11,\n",
       "   9,\n",
       "   29,\n",
       "   33,\n",
       "   18,\n",
       "   16,\n",
       "   16,\n",
       "   34,\n",
       "   39,\n",
       "   28,\n",
       "   12,\n",
       "   14,\n",
       "   24,\n",
       "   35,\n",
       "   13,\n",
       "   16,\n",
       "   12,\n",
       "   25,\n",
       "   26,\n",
       "   20,\n",
       "   13,\n",
       "   11,\n",
       "   10,\n",
       "   25,\n",
       "   22,\n",
       "   32,\n",
       "   19,\n",
       "   18,\n",
       "   15,\n",
       "   9,\n",
       "   27,\n",
       "   10,\n",
       "   20,\n",
       "   15,\n",
       "   19,\n",
       "   29,\n",
       "   14,\n",
       "   14,\n",
       "   13,\n",
       "   14,\n",
       "   11,\n",
       "   15,\n",
       "   14,\n",
       "   16,\n",
       "   46,\n",
       "   14,\n",
       "   15,\n",
       "   18,\n",
       "   14,\n",
       "   30,\n",
       "   22,\n",
       "   15,\n",
       "   17,\n",
       "   22,\n",
       "   13,\n",
       "   13,\n",
       "   33,\n",
       "   14,\n",
       "   12,\n",
       "   18,\n",
       "   15,\n",
       "   20,\n",
       "   41,\n",
       "   12,\n",
       "   16,\n",
       "   33,\n",
       "   24,\n",
       "   13,\n",
       "   18,\n",
       "   14,\n",
       "   11,\n",
       "   15,\n",
       "   32,\n",
       "   20,\n",
       "   16,\n",
       "   27,\n",
       "   21,\n",
       "   12,\n",
       "   25,\n",
       "   31,\n",
       "   11,\n",
       "   21,\n",
       "   13,\n",
       "   12,\n",
       "   21,\n",
       "   19,\n",
       "   21,\n",
       "   54,\n",
       "   12,\n",
       "   18,\n",
       "   14,\n",
       "   44,\n",
       "   66,\n",
       "   37,\n",
       "   16,\n",
       "   17,\n",
       "   13,\n",
       "   38,\n",
       "   33,\n",
       "   17,\n",
       "   14,\n",
       "   24,\n",
       "   33,\n",
       "   26,\n",
       "   25,\n",
       "   43,\n",
       "   25,\n",
       "   19,\n",
       "   14,\n",
       "   20,\n",
       "   11,\n",
       "   72,\n",
       "   16,\n",
       "   37,\n",
       "   11,\n",
       "   12,\n",
       "   16,\n",
       "   12,\n",
       "   20,\n",
       "   19,\n",
       "   25,\n",
       "   10,\n",
       "   41,\n",
       "   10,\n",
       "   29,\n",
       "   39,\n",
       "   10,\n",
       "   17,\n",
       "   11,\n",
       "   19,\n",
       "   24,\n",
       "   16,\n",
       "   18,\n",
       "   13,\n",
       "   24,\n",
       "   18,\n",
       "   14,\n",
       "   21,\n",
       "   22,\n",
       "   13,\n",
       "   14,\n",
       "   13,\n",
       "   15,\n",
       "   18,\n",
       "   12,\n",
       "   52,\n",
       "   17,\n",
       "   17,\n",
       "   14,\n",
       "   11,\n",
       "   27,\n",
       "   60,\n",
       "   23,\n",
       "   11,\n",
       "   15,\n",
       "   25,\n",
       "   21,\n",
       "   56,\n",
       "   17,\n",
       "   26,\n",
       "   15,\n",
       "   26,\n",
       "   21,\n",
       "   15,\n",
       "   13,\n",
       "   12,\n",
       "   11,\n",
       "   33,\n",
       "   24,\n",
       "   15,\n",
       "   17,\n",
       "   12,\n",
       "   17,\n",
       "   14,\n",
       "   15,\n",
       "   20,\n",
       "   9,\n",
       "   14,\n",
       "   15,\n",
       "   11,\n",
       "   15,\n",
       "   17,\n",
       "   22,\n",
       "   13,\n",
       "   26,\n",
       "   15,\n",
       "   19,\n",
       "   23,\n",
       "   27,\n",
       "   12,\n",
       "   22,\n",
       "   30,\n",
       "   18,\n",
       "   52,\n",
       "   48,\n",
       "   32,\n",
       "   20,\n",
       "   13,\n",
       "   20,\n",
       "   16,\n",
       "   20,\n",
       "   31,\n",
       "   13,\n",
       "   26,\n",
       "   37,\n",
       "   12,\n",
       "   11,\n",
       "   24,\n",
       "   10,\n",
       "   45,\n",
       "   17,\n",
       "   13,\n",
       "   12,\n",
       "   15,\n",
       "   17,\n",
       "   16,\n",
       "   35,\n",
       "   26,\n",
       "   11,\n",
       "   27,\n",
       "   16,\n",
       "   10,\n",
       "   10,\n",
       "   18,\n",
       "   28,\n",
       "   22,\n",
       "   36,\n",
       "   12,\n",
       "   13,\n",
       "   23,\n",
       "   11,\n",
       "   26,\n",
       "   22,\n",
       "   24,\n",
       "   17,\n",
       "   25,\n",
       "   13,\n",
       "   46,\n",
       "   29,\n",
       "   24,\n",
       "   21,\n",
       "   15,\n",
       "   14,\n",
       "   28,\n",
       "   8,\n",
       "   17,\n",
       "   27,\n",
       "   31,\n",
       "   40,\n",
       "   26,\n",
       "   24,\n",
       "   32,\n",
       "   60,\n",
       "   11,\n",
       "   15,\n",
       "   15,\n",
       "   14,\n",
       "   9,\n",
       "   18,\n",
       "   26,\n",
       "   15,\n",
       "   24,\n",
       "   32,\n",
       "   22,\n",
       "   13,\n",
       "   22,\n",
       "   18,\n",
       "   14,\n",
       "   20,\n",
       "   12,\n",
       "   11,\n",
       "   38,\n",
       "   17,\n",
       "   18,\n",
       "   28,\n",
       "   22,\n",
       "   20,\n",
       "   15,\n",
       "   26,\n",
       "   33,\n",
       "   121,\n",
       "   12,\n",
       "   20,\n",
       "   20,\n",
       "   20,\n",
       "   23,\n",
       "   36,\n",
       "   8,\n",
       "   12,\n",
       "   42,\n",
       "   17,\n",
       "   13,\n",
       "   19,\n",
       "   35,\n",
       "   17,\n",
       "   19,\n",
       "   42,\n",
       "   21,\n",
       "   28,\n",
       "   12,\n",
       "   14,\n",
       "   13,\n",
       "   14,\n",
       "   19,\n",
       "   30,\n",
       "   11,\n",
       "   15,\n",
       "   17,\n",
       "   29,\n",
       "   25,\n",
       "   15,\n",
       "   12,\n",
       "   20,\n",
       "   82,\n",
       "   18,\n",
       "   12,\n",
       "   44,\n",
       "   12,\n",
       "   22,\n",
       "   11,\n",
       "   23,\n",
       "   14,\n",
       "   29,\n",
       "   18,\n",
       "   42,\n",
       "   14,\n",
       "   50,\n",
       "   14,\n",
       "   22,\n",
       "   20,\n",
       "   20,\n",
       "   9,\n",
       "   25,\n",
       "   12,\n",
       "   14,\n",
       "   9,\n",
       "   8,\n",
       "   18,\n",
       "   17,\n",
       "   23,\n",
       "   23,\n",
       "   15,\n",
       "   11,\n",
       "   21,\n",
       "   14,\n",
       "   13,\n",
       "   9,\n",
       "   17,\n",
       "   21,\n",
       "   20,\n",
       "   11,\n",
       "   14,\n",
       "   13,\n",
       "   40,\n",
       "   17,\n",
       "   23,\n",
       "   17,\n",
       "   10,\n",
       "   11,\n",
       "   11,\n",
       "   28,\n",
       "   16,\n",
       "   12,\n",
       "   14,\n",
       "   12,\n",
       "   16,\n",
       "   14,\n",
       "   21,\n",
       "   16,\n",
       "   35,\n",
       "   17,\n",
       "   17,\n",
       "   20,\n",
       "   15,\n",
       "   20,\n",
       "   49,\n",
       "   10]},\n",
       " 'sampler_perf': {'mean_env_wait_ms': 0.04731069697601887,\n",
       "  'mean_raw_obs_processing_ms': 0.06725382557542246,\n",
       "  'mean_inference_ms': 1.4626616880962542,\n",
       "  'mean_action_processing_ms': 0.03731505540948616},\n",
       " 'off_policy_estimator': {},\n",
       " 'num_healthy_workers': 1,\n",
       " 'timesteps_total': 8000,\n",
       " 'timers': {'sample_time_ms': 13038.195,\n",
       "  'sample_throughput': 613.582,\n",
       "  'learn_time_ms': 1469.414,\n",
       "  'learn_throughput': 5444.346,\n",
       "  'update_time_ms': 1.691},\n",
       " 'info': {'learner': {'default_policy': {'cur_kl_coeff': 0.20000000298023224,\n",
       "    'cur_lr': 0.0003000000142492354,\n",
       "    'total_loss': 211.58801,\n",
       "    'policy_loss': -0.030461444,\n",
       "    'vf_loss': 211.61389,\n",
       "    'vf_explained_var': 0.0045384415,\n",
       "    'kl': 0.022958973,\n",
       "    'entropy': 0.67027843,\n",
       "    'entropy_coeff': 0.0}},\n",
       "  'num_steps_sampled': 8000,\n",
       "  'num_steps_trained': 8000},\n",
       " 'done': False,\n",
       " 'episodes_total': 381,\n",
       " 'training_iteration': 1,\n",
       " 'experiment_id': 'd4818098f4e84249aaf976a336fe534c',\n",
       " 'date': '2020-11-21_11-18-11',\n",
       " 'timestamp': 1605975491,\n",
       " 'time_this_iter_s': 14.520432233810425,\n",
       " 'time_total_s': 14.520432233810425,\n",
       " 'pid': 12711,\n",
       " 'hostname': 'MacBook-Pro',\n",
       " 'node_ip': '192.168.0.23',\n",
       " 'config': {'num_workers': 1,\n",
       "  'num_envs_per_worker': 1,\n",
       "  'create_env_on_driver': False,\n",
       "  'rollout_fragment_length': 200,\n",
       "  'batch_mode': 'truncate_episodes',\n",
       "  'num_gpus': 0,\n",
       "  'train_batch_size': 8000,\n",
       "  'model': {'fcnet_hiddens': [64, 64],\n",
       "   'fcnet_activation': 'tanh',\n",
       "   'conv_filters': None,\n",
       "   'conv_activation': 'relu',\n",
       "   'free_log_std': False,\n",
       "   'no_final_linear': False,\n",
       "   'vf_share_layers': True,\n",
       "   'use_lstm': False,\n",
       "   'max_seq_len': 20,\n",
       "   'lstm_cell_size': 256,\n",
       "   'lstm_use_prev_action_reward': False,\n",
       "   '_time_major': False,\n",
       "   'framestack': True,\n",
       "   'dim': 84,\n",
       "   'grayscale': False,\n",
       "   'zero_mean': True,\n",
       "   'custom_model': None,\n",
       "   'custom_model_config': {},\n",
       "   'custom_action_dist': None,\n",
       "   'custom_preprocessor': None},\n",
       "  'optimizer': {},\n",
       "  'gamma': 0.99,\n",
       "  'horizon': None,\n",
       "  'soft_horizon': False,\n",
       "  'no_done_at_end': False,\n",
       "  'env_config': {},\n",
       "  'env': 'CartPole-v1',\n",
       "  'normalize_actions': False,\n",
       "  'clip_rewards': None,\n",
       "  'clip_actions': True,\n",
       "  'preprocessor_pref': 'deepmind',\n",
       "  'lr': 0.0003,\n",
       "  'monitor': False,\n",
       "  'log_level': 'WARN',\n",
       "  'callbacks': ray.rllib.agents.callbacks.DefaultCallbacks,\n",
       "  'ignore_worker_failures': False,\n",
       "  'log_sys_usage': True,\n",
       "  'fake_sampler': False,\n",
       "  'framework': 'tfe',\n",
       "  'eager_tracing': False,\n",
       "  'explore': True,\n",
       "  'exploration_config': {'type': 'StochasticSampling'},\n",
       "  'evaluation_interval': None,\n",
       "  'evaluation_num_episodes': 10,\n",
       "  'in_evaluation': False,\n",
       "  'evaluation_config': {},\n",
       "  'evaluation_num_workers': 0,\n",
       "  'custom_eval_function': None,\n",
       "  'sample_async': False,\n",
       "  '_use_trajectory_view_api': False,\n",
       "  'observation_filter': 'NoFilter',\n",
       "  'synchronize_filters': True,\n",
       "  'tf_session_args': {'intra_op_parallelism_threads': 2,\n",
       "   'inter_op_parallelism_threads': 2,\n",
       "   'gpu_options': {'allow_growth': True},\n",
       "   'log_device_placement': False,\n",
       "   'device_count': {'CPU': 1},\n",
       "   'allow_soft_placement': True},\n",
       "  'local_tf_session_args': {'intra_op_parallelism_threads': 8,\n",
       "   'inter_op_parallelism_threads': 8},\n",
       "  'compress_observations': False,\n",
       "  'collect_metrics_timeout': 180,\n",
       "  'metrics_smoothing_episodes': 100,\n",
       "  'remote_worker_envs': False,\n",
       "  'remote_env_batch_wait_ms': 0,\n",
       "  'min_iter_time_s': 0,\n",
       "  'timesteps_per_iteration': 2000,\n",
       "  'seed': None,\n",
       "  'extra_python_environs_for_driver': {},\n",
       "  'extra_python_environs_for_worker': {},\n",
       "  'num_cpus_per_worker': 0,\n",
       "  'num_gpus_per_worker': 0,\n",
       "  'custom_resources_per_worker': {},\n",
       "  'num_cpus_for_driver': 1,\n",
       "  'memory': 0,\n",
       "  'object_store_memory': 0,\n",
       "  'memory_per_worker': 0,\n",
       "  'object_store_memory_per_worker': 0,\n",
       "  'input': 'sampler',\n",
       "  'input_evaluation': ['is', 'wis'],\n",
       "  'postprocess_inputs': False,\n",
       "  'shuffle_buffer_size': 0,\n",
       "  'output': None,\n",
       "  'output_compress_columns': ['obs', 'new_obs'],\n",
       "  'output_max_file_size': 67108864,\n",
       "  'multiagent': {'policies': {},\n",
       "   'policy_mapping_fn': None,\n",
       "   'policies_to_train': None,\n",
       "   'observation_fn': None,\n",
       "   'replay_mode': 'independent'},\n",
       "  'logger_config': None,\n",
       "  'replay_sequence_length': 1,\n",
       "  'use_critic': True,\n",
       "  'use_gae': True,\n",
       "  'lambda': 1.0,\n",
       "  'kl_coeff': 0.2,\n",
       "  'sgd_minibatch_size': 128,\n",
       "  'shuffle_sequences': True,\n",
       "  'num_sgd_iter': 2,\n",
       "  'lr_schedule': None,\n",
       "  'vf_share_layers': False,\n",
       "  'vf_loss_coeff': 1.0,\n",
       "  'entropy_coeff': 0.0,\n",
       "  'entropy_coeff_schedule': None,\n",
       "  'clip_param': 0.3,\n",
       "  'vf_clip_param': 10.0,\n",
       "  'grad_clip': None,\n",
       "  'kl_target': 0.01,\n",
       "  'simple_optimizer': True,\n",
       "  '_fake_gpus': False},\n",
       " 'time_since_restore': 14.520432233810425,\n",
       " 'timesteps_since_restore': 0,\n",
       " 'iterations_since_restore': 1,\n",
       " 'perf': {'cpu_util_percent': 13.033333333333333,\n",
       "  'ram_util_percent': 59.18095238095237}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# run one training iteration\n",
    "# https://github.com/ray-project/ray/blob/master/rllib/agents/ppo/ppo.py\n",
    "env_name = 'CartPole-v1'\n",
    "\n",
    "ppo_config = DEFAULT_CONFIG.copy()\n",
    "if n_gpus:\n",
    "    ppo_config['num_gpus'] = n_gpus\n",
    "    ppo_config['tf_session_args']['device_count']['GPU'] = n_gpus\n",
    "\n",
    "ppo_config['num_workers'] = 1\n",
    "ppo_config['num_sgd_iter'] = 2\n",
    "ppo_config['sgd_minibatch_size'] = 128\n",
    "ppo_config['lr'] = 0.0003\n",
    "ppo_config['gamma'] = 0.99\n",
    "ppo_config['model']['fcnet_hiddens'] = [64, 64]\n",
    "ppo_config['timesteps_per_iteration'] = 2000\n",
    "ppo_config['train_batch_size'] = 8000\n",
    "ppo_config['num_cpus_per_worker'] = 0  # This avoids running out of resources in the notebook environment when this cell is re-executed\n",
    "ppo_config['framework']='tfe'\n",
    "agent = PPOTrainer(ppo_config, env_name)\n",
    "result = agent.train()\n",
    "\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 19.2/32.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/16 CPUs, 0/0 GPUs, 0.0/10.5 GiB heap, 0.0/3.61 GiB objects<br>Result logdir: /Users/drucev/ray_results/cartpole_test<br>Number of trials: 16/16 (16 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc  </th><th style=\"text-align: right;\">  gamma</th><th style=\"text-align: right;\">    lr</th><th>model/fcnet_hiddens  </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">   ts</th><th style=\"text-align: right;\">  reward</th><th style=\"text-align: right;\">  episode_reward_max</th><th style=\"text-align: right;\">  episode_reward_min</th><th style=\"text-align: right;\">  episode_len_mean</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00000</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">0.0003</td><td>[16, 16]             </td><td style=\"text-align: right;\">    37</td><td style=\"text-align: right;\">         223.382</td><td style=\"text-align: right;\">37000</td><td style=\"text-align: right;\">  300.93</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  44</td><td style=\"text-align: right;\">            300.93</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00001</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.999</td><td style=\"text-align: right;\">0.0003</td><td>[16, 16]             </td><td style=\"text-align: right;\">    35</td><td style=\"text-align: right;\">         209.194</td><td style=\"text-align: right;\">35000</td><td style=\"text-align: right;\">  301.4 </td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  18</td><td style=\"text-align: right;\">            301.4 </td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00002</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">0.0001</td><td>[16, 16]             </td><td style=\"text-align: right;\">    39</td><td style=\"text-align: right;\">         232.797</td><td style=\"text-align: right;\">39000</td><td style=\"text-align: right;\">  301.73</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  26</td><td style=\"text-align: right;\">            301.73</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00003</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.999</td><td style=\"text-align: right;\">0.0001</td><td>[16, 16]             </td><td style=\"text-align: right;\">    46</td><td style=\"text-align: right;\">         280.03 </td><td style=\"text-align: right;\">46000</td><td style=\"text-align: right;\">  302.52</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                 101</td><td style=\"text-align: right;\">            302.52</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00004</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">0.0003</td><td>[32, 32]             </td><td style=\"text-align: right;\">    35</td><td style=\"text-align: right;\">         214.664</td><td style=\"text-align: right;\">35000</td><td style=\"text-align: right;\">  306.52</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  17</td><td style=\"text-align: right;\">            306.52</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00005</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.999</td><td style=\"text-align: right;\">0.0003</td><td>[32, 32]             </td><td style=\"text-align: right;\">    35</td><td style=\"text-align: right;\">         212.746</td><td style=\"text-align: right;\">35000</td><td style=\"text-align: right;\">  307.41</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  42</td><td style=\"text-align: right;\">            307.41</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00006</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">0.0001</td><td>[32, 32]             </td><td style=\"text-align: right;\">    36</td><td style=\"text-align: right;\">         220.485</td><td style=\"text-align: right;\">36000</td><td style=\"text-align: right;\">  303.03</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  23</td><td style=\"text-align: right;\">            303.03</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00007</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.999</td><td style=\"text-align: right;\">0.0001</td><td>[32, 32]             </td><td style=\"text-align: right;\">    44</td><td style=\"text-align: right;\">         273.047</td><td style=\"text-align: right;\">44000</td><td style=\"text-align: right;\">  302.42</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  74</td><td style=\"text-align: right;\">            302.42</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00008</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">0.0003</td><td>[64, 64]             </td><td style=\"text-align: right;\">    33</td><td style=\"text-align: right;\">         268.542</td><td style=\"text-align: right;\">33000</td><td style=\"text-align: right;\">  311.17</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  22</td><td style=\"text-align: right;\">            311.17</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00009</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.999</td><td style=\"text-align: right;\">0.0003</td><td>[64, 64]             </td><td style=\"text-align: right;\">    32</td><td style=\"text-align: right;\">         259.593</td><td style=\"text-align: right;\">32000</td><td style=\"text-align: right;\">  301.79</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  12</td><td style=\"text-align: right;\">            301.79</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00010</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">0.0001</td><td>[64, 64]             </td><td style=\"text-align: right;\">    37</td><td style=\"text-align: right;\">         290.015</td><td style=\"text-align: right;\">37000</td><td style=\"text-align: right;\">  303.51</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  45</td><td style=\"text-align: right;\">            303.51</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00011</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.999</td><td style=\"text-align: right;\">0.0001</td><td>[64, 64]             </td><td style=\"text-align: right;\">    36</td><td style=\"text-align: right;\">         283.418</td><td style=\"text-align: right;\">36000</td><td style=\"text-align: right;\">  310.99</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  85</td><td style=\"text-align: right;\">            310.99</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00012</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">0.0003</td><td>[128, 128]           </td><td style=\"text-align: right;\">    32</td><td style=\"text-align: right;\">         267.668</td><td style=\"text-align: right;\">32000</td><td style=\"text-align: right;\">  309.04</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  17</td><td style=\"text-align: right;\">            309.04</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00013</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.999</td><td style=\"text-align: right;\">0.0003</td><td>[128, 128]           </td><td style=\"text-align: right;\">    32</td><td style=\"text-align: right;\">         267.383</td><td style=\"text-align: right;\">32000</td><td style=\"text-align: right;\">  302.49</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  10</td><td style=\"text-align: right;\">            302.49</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00014</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.99 </td><td style=\"text-align: right;\">0.0001</td><td>[128, 128]           </td><td style=\"text-align: right;\">    33</td><td style=\"text-align: right;\">         257.99 </td><td style=\"text-align: right;\">33000</td><td style=\"text-align: right;\">  300.21</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  16</td><td style=\"text-align: right;\">            300.21</td></tr>\n",
       "<tr><td>PPO_CartPole-v1_268dd_00015</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">  0.999</td><td style=\"text-align: right;\">0.0001</td><td>[128, 128]           </td><td style=\"text-align: right;\">    32</td><td style=\"text-align: right;\">         249.885</td><td style=\"text-align: right;\">32000</td><td style=\"text-align: right;\">  301.46</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                  14</td><td style=\"text-align: right;\">            301.46</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-11-21 11:27:28,357\tINFO tune.py:439 -- Total run time: 556.59 seconds (556.31 seconds for the tuning loop).\n"
     ]
    }
   ],
   "source": [
    "# tune hyperparamters with grid search\n",
    "# https://github.com/ray-project/ray/blob/master/python/ray/tune/tune.py\n",
    "ray.init(ignore_reinit_error=True)\n",
    "env_name = 'CartPole-v1'\n",
    "ppo_config = {\n",
    "    \"env\": env_name,\n",
    "    \"num_workers\": 1,\n",
    "    'model': {\n",
    "        'fcnet_hiddens': tune.grid_search([\n",
    "                                           [16, 16], [32, 32], [64, 64], [128, 128],\n",
    "                                          ])\n",
    "    },        \n",
    "    'train_batch_size': 1000,\n",
    "    \"lr\": tune.grid_search([0.0003, 0.0001]),\n",
    "    'gamma': tune.grid_search([0.99, 0.999]),\n",
    "    'framework': 'tfe',\n",
    "    'num_gpus': n_gpus  \n",
    "}\n",
    "                      \n",
    "analysis = tune.run(\n",
    "    \"PPO\",\n",
    "    name='cartpole_test',\n",
    "    verbose=1,\n",
    "    stop={\"episode_reward_mean\": 300},  # stop when a parameter set is able to reach 300 timesteps\n",
    "    config = ppo_config,\n",
    "    checkpoint_freq=10,\n",
    "    checkpoint_at_end=True,\n",
    "    checkpoint_score_attr='episode_reward_mean',\n",
    "    num_samples=1,  # for grid search, number of times to run each hyperparameter combo\n",
    "    #     with_server=True,\n",
    "    #     server_port=8267,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABn7klEQVR4nO3dd3RURcPH8e9sT++9kEIooQUIvffee5MmiICggIgKNkTBjgpIU1CkCtJEpPfeewtJSID0Xrfd9w/yPK8+gKICgTCfczjszt7dnZlDfrnMnTsjFEVBkiRJKllUxV0BSZIk6eGT4S5JklQCyXCXJEkqgWS4S5IklUAy3CVJkkogTXFXAMDd3V0JCgoq7mpIkiQ9VY4fP56iKIrHvV57IsI9KCiIY8eOFXc1JEmSnipCiNj7vSaHZSRJkkogGe6SJEklkAx3SZKkEkiGuyRJUgkkw12SJKkEkuEuSZJUAslwlyRJKoFkuEuSdF+xNxaQkXn84X1g0kW4sP7hfZ50XzLcJekZl2/OJzE38a7y5OStXLv2IQm3f344X5SbAkt7wa8TwZj7cD5Tui8Z7pL0DMsx5vDithd5fsvzmCym/5bn5l7n/IUJODpUJixsyt/+3Pyc7D8WmAthRX/ISYReP4LO7t9WXfoLMtwl6RmVUZDB81ue53TSaUZGjESr1gJgNudw9txIVCodlSrNQq3WP/BnKlYr+1ctY96LQ0i9GVdUqMDGcXDjIHSeDf7VH0VzpP/xRKwtI0nS45WUl8TwLcOJy45jZtOZNPRvCICiKFy8OInc3CiqRizGYPB94M8syM1h3ScfEX/hBCpdWQrzDHdeOPAVnFoCjSZBxW6PojnSPchwl6RnTHx2PMO2DCOtII05zedQ06fmf1+7cWM+Scm/Urr0JFxd6z7wZyZej2L1B1PJz07D1q0FHcYMwDfMFS7/ClvfggpdoNFrKIqCNdeE2l5312ecjstArRJU9HN6KO181v3lsIwQwiCEOCKEOC2EOC+EeLeoPFgIcVgIcU0IsUIIoSsq1xc9v1b0etAjboMkSQ/oesZ1Bm4eSJYxiwUtF1DDuwY3V67i/IfTSUvbz7Woj/H0bEtgwPMP/JknNv/Gj2+OJz+ngFJVhzP4o5H4hxgg4Rysfh58I6DTbCw5JlIXnSd57hkUk+W/7y8wWZj+6yW6zN7PR79d/tttupp+9W+/52GLjY3ll19+QVGUB36POT2d22+9TfauXY+kTg9y5l4INFUUJUcIoQX2CSF+BcYBnyuKslwI8Q0wFJhT9He6oiilhRC9gRlAr0dSe0mSHtiF1AuM2DoClVDxXevvCFV5ETX2ZY5cOkWakyPljv+MvWMo5ctNRwgBQG5GIVqDGp3h7qgwG41s+Pwrrp/YiUobQMO+L1G9TXnEudWw+XVQqUDvgNJ7KXnnc8hYFwUWK06tg0B957zyxI10Xl11mqjkXCICnOlU5cGHgTIKMvjwyIdsit7ED21+IMIz4mF0099iNBrZvn07hw8fxtnZmZycHBwcHP70PYrVSsZPP5H86WdYcnLQlQqExo0fet3+MtyVO7+Kcoqeaov+KEBToG9R+WLgHe6Ee6eixwA/AV8LIYTyd36lSZL0UJ1MOsnIbSNx1Dkyv+V83C8ncnHiCxyzVZHq4oC2rhtq7Q0qVZyNRnNnJsutq+lsnn+ewPKuNB8c/ofPS7t1i5XvTSU3PQ47t7p0mzQKj0AnOPYtbHwF9A5gMWHpvZH0tRkUXEhFV8oRlx5l0LrbUGCy8NnmKyzYex1PBwNVA5w5GZdBiIcd3ar7/2V7tsZu5f1D75NVmMWLVV6kgluFR9JvfyYmJoZ169aRnp5OzZo1adasGXr9n198zj97joSpUyk4cwbbyEi83pqCoUyZR1K/BxpzF0KogeNAaWAWEAVkKIpiLjokHvAreuwHxAEoimIWQmQCbkDK/3zmcGA4QGBg4L9rhSRJ93Xw1kHG7hyLl60X85rMQf3tT1z59lsOlPYjV6fBuU0oIQEbqFjhG+zsQlAUhTM74tm/+hqO7gaqtvrjz+fxX7ex+/s5KAqE1hhEuzFd0OrUsO9z2PYOGJygIJPCml+RutSM1ZiGU9tg7Ov7IVSCYzFpTPzpDNdTcmlVwYuzNzM5ezOTKe3DGVIv6E/bkpqfygeHP2BL7BbKu5ZnXot5lHUt++g67x6MRiPbtm3jyJEjuLi4MGjQIP5qJzlLRgZJX3xBxoqVqN3c8P1oBo4dOvz3f0iPwgOFu6IoFiBCCOEM/AyU+7dfrCjKPGAeQGRkpDyrl6RHYFfcLsbtGkewUzCzy71F3gvjSbl4kb3hwRjVKrw6BxLotoHSoa/h4dECU6GFnUsucfVoIsFV3Gk2KBy9zZ2YMBYW8PP0mcRf2Ita50uzoa9QqXH5O1Mdt71zJ9w1ehSLiRzvD8jcE4zW34BHz7JoPW2JScll9q5rrDoej6+TDQNqB7L8aByeDgZWjahD1UCX+7ZDURQ2x2zmw8MfkmPK4aWqLzG44mC0Ku3j6cgi0dHRrF+/nvT0dGrVqkWzZs3Q6e6+OPwf1oICMtasIeWrr7FkZuIyoD8eL72E+i+Gbh6GvzVbRlGUDCHETqAO4CyE0BSdvfsDN4sOuwkEAPFCCA3gBKQ+xDpLkvQANkdv5vW9r1PerTyf2w0io/cQ0mz17KlUGpNWS6muHvjab6RMmbcJ8H+OjMQ8fp17lrTbudTqFEL1VqUQqjtnlnEXr7H2ow8x5iXi4t+A7m+MxtHNDqxW+GUcHP8OEFgNPqQUvI4xzh/HVoE4NAzgcnI2s5edZOOZW2jUKvrWDCQpq4AfDt2gWTlPPu1ZBWfb+wdkfHY8Hx39iJ1xO6nkXon36r5HaZfSj6kX74iLi2Pv3r1cuXIFV1dXBg8eTKlSpe57vDktjfQfl5K+dCmW9HRsqlfHe8pkDOX+9XnxA/vLcBdCeACmomC3AVpw5yLpTqA7sBwYCKwresv6oucHi17fIcfbJenx+vnqz7x94G2qeVXj89CJJPUbTIK/D4fsNeTb2lG2uw2e2t8oX34Gvj7diT6TwrbvLiBU0OGlKgSGuwF3zph3/bCGE5t+ALRUbfcSTQa0vDOcYDHBmuFwfg0ARsd6JCeNRu3lieeQslywmPl6yXG2XUzETqdmWIMQ6oe5M3ntOeLT83m9TTmGNQhBpbr30ES2MZv5Z+ez5MISNCoN46qPY0D4ADSqxzODW1EUoqKi2LdvHzExMdjY2NC4cWPq1q1737N1Y0wMqYsWkfnzWpTCQuybNMFtyGBsIiMf6RDMvTxIL/kAi4vG3VXASkVRNgohLgDLhRDvAyeBhUXHLwR+EEJcA9KA3o+g3pIk3cePF39k+pHp1POtx6c13yeh3yBu2eo5ZivIdPWkatdC3MQeKlT4Ek+P1hzZGM3RjdF4BDrQenhFHN1tAMhMymD1h5+QfusUevtgOk94Df/yRRc7jbmwrC9E7wIgR9ufjKSeGOr5c6KULd9vOs++ayk42Wh5uXkYg+oGsftKMsO+P4azjY4Vw2sTGeR6z/qbrWZWX1nNrFOzyCjMoGNoR16q+hJedl6Po/uwWq1cunSJvXv3cvv2bRwcHGjVqhXVqlW75wVTS0YGuYePkLVxA9nbtiO0Wpw6dcJ18CD0ISGPpc738iCzZc4AVe9Rfh2oeY/yAqDHQ6mdJEl/y4KzC5h5YibNApsxo/50kl6eQHp8PIcrBJPl6klk9yxclNNUqjgHZ6eGbP32AlePJlK2tjeN+5ZFo1NjsVg5sOoAR9d/g2LJJLBSWzpPHIZWVzS+nXoNFnVEyb4JwkCqcTzXNfXZVsWOtWeiSdpfiIeDntfblKNf7VIYNCo+/PUSC/dFUzPYldn9quFuf+9ZJXvj9/LpsU+Jyowi0iuSV2u8Srhb+D2Pfdjy8/M5ffo0R48eJTU1FRcXFzp06ECVKlXQaP4/Kq35+eQdP0HeoYPkHjhIwcWLoCionZxwG/ECrv36oXF3fyx1/jPyDlVJKiF+vPgjM0/MpF1IO96v9z7ps74ha8d2fqtZBbXVRM02N3HmBlWqfIteXY11n58k4XoWtTuHUK1VKYQQxJxLYeu8H8lK3I5GZ0/Lke9Qvv7v1oI5/A1sfgMUC0ZVGTbmj2aZnT/HctJRnUmnSVlPetUIoEk5T7RqFWm5RoZ/f4wDUakMqhvEm+3Ko1Xffe9kgbmA1/e+zrYb2wh0CGRmk5k0CWjyWIYybt26xdGjRzl79ixmsxk/Pz+6detGeHg4arUauDNEk7N7N2mLFpN//DiKyQRaLbZVquD+0mjsatfBplJFhPbxXuD9MzLcJakE2BW3i4+OfkSTgCZMqzeN3G07SJk1i83Vq2AoyCGwTSEu9ilEVPkec25pfpp1jLwsI62GVaR0dU/SE3LZvew00ceXYzVdxyesKp1fexVbB8c7X5CTDEt7wq0TKELLTkYxLq8WGQgCdSpebVWWbtX88XYy/LdO525m8sIPx0nOKeTj7pXpERlwz7rnmfIYs3MMR24fYWy1sQwMH/jfRcweFaPRyPnz5zl69Ci3bt1Cq9VSuXJlIiMj8fX9441UecePk/TZ5+QfP47W3x+X5wZgV7sOttWrobK1faT1/DdkuEvSU+586nkm7plIedfyTG8wnfyrUURPfI2zoaFgzsG9UgFeoZlUjVhKepwnv807jkanpsv4angGOnBkYzRH1u3DmPMLUECjAcOo3q5j0UVTMxyaDdvfA6uJJNvqjM4YzRGrnpq+Trzcthy1Q9zuuii67tRNXlt9BmcbHateqEOVAOd71j3XlMuo7aM4mXSSafWn0SG0wyPtK6vVyqlTp9ixYwc5OTl4eHjQpk0bqlSpgsFg+MOxBZcvk/zZ5+Ts3o3GwwPvd97BuVvXJ+rs/M/IcJekp9jtnNuM3j4aF70LXzf7GnNWAWeHj6DQxkCiuwEb+2yC6+dRvdpyoo8b2LP8DK4+drQbVRmDnZbN885y5eAmzAX7cfL0psMr7+MVUhry0uD0Mjg8FzJisaDiG/v3+CylNI5C8HGbcnRvGHzXsEme0cz7v1xk6eEb1AxyZVa/ang43Ht8PduYzYvbXuRcyjlmNJhB6+DWj7Svrl+/zm+//UZiYiL+/v5069aNoKCgP7RBURSM0dGkzPmGrI0bUTk44DF+HK79+6OysXmk9XvYZLhL0lMq25jNyO0jKTQXMr/NfKxmA9tGDCckJYkTTWqgykikTNtCqlZbytF1hZzfc5lSldxoObQCxnwLaz4+wq3Lq7EaL1G2TgNaDh+NLvkMrPkIzq8FSyGKUHNSVOBFy5skpkBnO1umjKiJm8fdm22cjsvg5RWniEnNZViDYCa2LnfP8XWAzMJMRmwdwaX0S3zS6BOal2r+yPopJSWFLVu2cOXKFZydnenevTvlQ0MxXrtG1saNGKOjMcbEUBgTgzEmFiUvD2Ew4DZsGG5Dh6B2ejpXqZThLklPIZPVxPhd44nJjGFOizlor6RzbtIIyt++yfEOjTHeiCOsjZkqEYv59eskkm9kU7VFILW7hJJ8I5uNXx8kK2E1VtMtGnbvTmRAPuK7ZpB8EXT24OBFYnomb6gnsj03mFBUfBDoQZOhVVHp1X+oi9liZc6uKGZuv4qHg54fn69F3dD7zxZJL0hn+NbhRGVE8UXjL2gU0OiR9FFubi67d+/m2LFjaDQamjdvTlUPD7JXruLaunVYc4qWzFKp0Pr5oQsKwjYyEl1QEA7Nm6P19Hwk9XpcZLhL0lNGURSmHZrGwdsHmVptMq4LtpOz9EfULm5cHdOHxN2H8aoI5SrNYd2nsQC0fbESwVU8uHoska0L91CY9TNCFND1hYEEn/sAzl8Hv+oQORTL2Z9ZnFaZTyy9sZrUjETHwGoBeHYLQ/zPmfiN1DxeWXmK47HpdKziy9ROFXGyvf+YdEp+CsO2DCMuO46vmn5FPb96D71/srOzOXDgAMeOHcNsNlMtIoIaQlA4dx43jh5FaLU4tG6NQ4vm6ENC0AYGovqTJQSeVjLcJekps/DcQlZfXc1EdVvKvrIAS3w8vzVpScX+NYme/wO2bloCwt5j6/zY/96Y5OBm4MjGaA6t2YY5/1dsHe3p8fwQ3Ha9AlYT9F4G51Zz/sg2XhdvcsboQ10bA6/kayjTtBSOLUrdNTb90/F43ll/HpVKMLN3BJ0i/O5bZ6PFyLJLy5h7Zi5mq5lZzWZRy6fWQ+2XzMxM9u/fz/Hjx7FarYQHBlIpPR3Vx5+QnpKC1t8fzwnjceraFY3rvW+gKklkuEvSU2TN1TXMPfgF7x73p/zu9dz28GL+a+/yfEUj52d/D1Ydzt4jOLcrnQoN/ajfozT52SbWf3mKmJNbMefvxSs4lG69m2CzaQTYOEOjyeRvmMgXmQ1ZYPkAZ4OOqXYGGhcI3HqXxTbij8MTqTmFvPHzWX47n0itYFc+6xWBn/O9LzYqisKW2C18cfwL4nPiqe9XnwmREwh1Dn1ofZKWlsaerds4c+nOzUShmVmUOXoU+5QUFCGwadwYlz69satfH6F6draNluEuSU+JzTGbWbj2bb5aq8Ux7QZrWrTjt+79eE9/gtNfrwJhwNblObJS7WkxpCxhNby4sO8We1eeoiBzG5bCq5SpXY82jYPQrB8K7mFQpg0H1s7jNcsk4iwudC/tydDYQpwMWtxHhKML+OPqhVsvJPL6mjNk5Zt5o205htYPQX2ftWFOJ5/mk6OfcCr5FGEuYcxtMZe6vg++dd+fMd26RdLefew7e4ZLajXCaiXk+nXKX72Ga3AQNm3bYFOlCraRkWi9Hs+yBU8aGe6S9BTYE7+HVQsmMm2DFbW9HeMnvE5epcq8b9zBidkbUKls0NoOwODgTbuRVdDoVKz74iQ3zh7EUrgLgZmG/YcQ6XYbsW4EBNTGYufJV7uimWl+nSA3W74N9aHM0RS0fva4DwhH7fT/UxizC0xM3XiBlcfiKe/jyI/PR1DW+97L1sZnxzPzxEw2x2zG3cadd+u+S6fQTqhV6nse/yDMaWnkHTpE7qHDZB49wjlbWy6VK4dFraa80Ujt0qXx6NYNfXh4iRw//ydkuEvSE+7Y7aNs/2A043aayCkdxrAhL+Mb4M/UnF84vnALao0tat0AXP18aT+yMtFnUti36jSFWVsxF17DJ6wsrUaMwe3CAtjyJZRuQWpGOi9HhbPXWpkuVXyZoBhQjqRgU8UD1+5hCO3/B/GR6DTGrTzFrYx8RjYO5eXmZdBp7h7eyCjIYN7ZeSy7tAyN0PBC5RcYUnEIttp/dhenOTWV7C1byPp1M3lHj2IFYsuV41xkJHlqNWX8A2jZqSPuHh7/tGtLNBnukvQEO3/zJKdeGUrvMyau1q3LS72G09TXkyGJKzj23R40egeEph9+5QJp3K8sW767QNzZA1gKdyNUZhr1H0K1ulVQbXwJondD2XYci0pgdO4Q0lQuTGsXTvMzGZjiUnBsWQqHJgH/vXBqslj5dMsV5u6JIsDFlpUv1LnnSo4F5gKWXlrKgjMLyDXn0rl0Z0ZWGfmPVnE0p6eTvXUrWb/+St7hI2C1ogsJIW/oUA6pBMlZWfj7+9O7ZUu5g9tfkOEuSU+oa1HHiH5hMPXizfzWtQvTm/dgbCkvGl6ey+ElR9EanBCavoTVCKZmh2DWf3GItLh1WIxR+JYpT6sXRuF6YwPMGQkqDUp4VxaeKWC6+WX8nA2s6hCB16ZYTJlGXPuVw7bS/58BJ2YVMHrpCY7GpNOnZgCT24Vjp/9jXFgVKxuvb+Srk1+RkJtAI/9GvFzt5b+9kYaiKOQdPUrad4vI2bMHLBa0pQJxGz4MS/0G7Lh8iatXr+Li4kKPHj0IDw9/7GujP41kuEvSEyjm+C4SR44iIM/KzOFD2Vy9BbPK+mG/9Q0O/XITna0bQtObyk3DKFfXm9Uf7SI7cSUomTQaMJRq1YJRbRgIt09BSBOyrTomnArkN2tNWpVzY1q9MhiXX8aqgMewSuhLOf73uw9cS2HM8pPkGS33neJ4PeM6k/ZO4mLaRSq4VWBavWnU9LlrBfA/pVgsZG/dSurCbyk4exa1iwtugwfh2LYtIjSUvXv3cvDXTajValq0aEGtWrX+sPSu9OdkT0nSEyZm/xZSR70MWpg47mUSytRjZVl3or97gXPHCjE4BqCoOlG7c3m8gx1ZM2Mzeemr0eqgy7jJ+CdthPlDwdYNIvpz7cxBhueP4gZeTG5bjr4ujqQtvojaSYf74IpoizbnsFoV5uyO4tMtlwnxsGfZsGqEed190XRLzBam7J+CQWP475owKvHgUwyt+flk/PwzaYsWY7pxA21gIN7vvI1T584IvZ6zZ8+y9euvyc7OpkqVKjRv3hyHx7DnaEkjw12SniDXdq4ne+wksu0Er4x5Fc+wOqzyt7L3kyFkRKuwdY3AShOa9i+PzqBh7WfrMGZvwNbJkV4j+uCyazikRUG5dpAZz47j5xhrmYLOxoYf+9cg/HYBaUsvofN3wG1gOGr7OzNLMvKMvLLiFDsvJ9Ohii/Tu1a6axjGbDXz5Ykv+e78d1T2qMynjT7F2877gdtmycklfckPpC3+Hkt6OoYqlfEcPx6H5s0QajU3b95k8+bNxMXF4evrS8+ePQkIuPcywdJfk+EuSU+IS7+tpHDC26Q6qRk3ZhK1K9Rnou4qW979hPx0DXbuzRHqCNoMq0h2agG/LFyOOXcb7oGl6NW1KoaNA+/clFShC8r59cwWPfnE1J4Kvk5807869gcSyNx3E0MFN1x7lUWluzMj5nRcBiN/PEFSdgFTO1Wgf+1Sd41pp+anMnHPRI4kHKFX2V5MrDERnfrBphxac3NJ+3EpaQsXYsnMxL5RI9yGPY9N9TubgERFRXHgwAGuX7+Ora0tHTt2JCIiAtUzdMPRoyDDXZKeAOc3/IBl0gckuGkYN3YKHSrVpU/KJn6Z+zNWiw471+7obENoVzTV8dCaZVgKDlGqYhW6VDWi/u0V8K4M+enkndvERPsZbEz1pWMVXz5sH07+mmvkXEzDvq4vTu1DECqBoigs3BfNjM2X8HQwsGpEXSLuse76meQzjNs1jozCDN6v9z6dSnd6oDZZ8/JIX7aM1AULsaSnY9eoIR6jR2NTqRIWi4WzZ8+yf/9+EhMTsbe3p3nz5kRGRt61rrr0z8hwl6Ridmb1fMSUz4j30jJu7LsMqlSDWidnsn31ebS2Nmht+2Hv7kvr4RU5siGKKweWYTGeJ6JhHZraHkAc2wfBjSD2AHG25Rlu/zGX0qy83qYcQyr6krrgHObkPJw7hGBf787F0bRcIxNWnWbHpSRahnvxUffKONv+8UxcURSWX17OR0c/wsvWix/a/EB5t/J/2R5rQQHpy5eTOn8BltRU7OrXx2P0KGwiIjAajRw6dIiDBw+SmZmJu7s7nTp1olKlSvJi6UMme1OSitHJpV+hnTqbaF8d48a8z5jwULzXj+TEISt27u6Yzb3wDPKifs8wtiw4S+K1VVhNV2nWth5VUpcg0lMgsC5E7+agdz9GJXXCZIXvBlWntkZH8qxTKAq4D66IIcwFgMPXUxm7/BRpuUbe7ViB5+rcPQwTmxXLuwff5WjCUer51WNGgxk46f98XXPFbCZz3TqSv/oac0ICtnVq4/HSl9hWqwZAUlISK1asIDU1lcDAQNq2bUtYWJgcfnlEZLhLUjE5uugTbGcs5HKAntfGTGdcgBHdd6O4EWWHi384eTktCK7iRfm6Pmz48hh5aeuwmq7TrW05gm58Drau4ByIEnuAxcEzmHolkGB3A/MGVMczKouU9VfQuBlwG1gBrbsNFqvC1zuuMXP7FUq52bFmYF0q+v0xsE1WE9+f/545p+egU+l4u87bdA3r+qezYRRFIWfHDpI++xxjVBSGypXxnTEDu1r/PzXyzJkzbNiwAZ1Ox4ABAwgNfXgLh0n3JsNdkorBwTnv4ThzGeeDbZjy0gzG2e2HBXvJSLLD2bcpeTlVqNzEHztnPb9+cxxr4QYwRjO4iQHX6PngVREy4ygoLGRKwFJWXYLm5T35rHslLFvjyDh0G0NZF1z7lEPo1ZyNz2Tapgscup5Gl6p+TO1cEfv/mQ1zPvU87xx4h0tpl2gW2Iw3ar2Bp+2fb1iRd+wYSZ9+Rv7Jk+iCgvD7ciYOLVr8938CZrOZLVu2cOTIEQIDA+nRo4ec1viYyHCXpMds/xdv4PrNz5wIs2PqqI8Ykzcf8+o0zAX22Lp1wGINo9mgMsRdSOPMjmuolI3YWa7RNzIZQ0I0BNWHmH0kutbkBWUSp64VMKZZGC/VDCTjx8sUXs/EvpE/6dU9+HFfNOtO3+R6ci62OjWf9KhC9+r+f6hPvjmfOafmsPjCYlwNrnze+PO/3PbOlJhIwrvvkbNjx53No999987m0b8bN8/MzGTVqlXEx8dTp04dmjdvjlr9zxcPk/4eGe6S9Bjtmf4KHos2c7i8I5+M/ICxN2dQuF2LSuWKxqYjLr6hNOpTlr0rrpAUm4BGtRF/5Sxty0SjMukgoi+cWsrxUs8z4mZrco0m5vSrRkOTiqTPT6KYrZyNcOGbqJuc3n0BIaBWsCvDGoTQpqL3Hy6a/u/yAd3CujEuchyOOsc/aQFkb9vG7TcnYzUa8Xj5ZVwHPnfX5tFRUVGsXr0as9lMjx49qFChwiPpT+n+/jLchRABwPeAF6AA8xRFmSmEeAcYBiQXHfqGoiibit7zOjAUsABjFEX57RHUXZKeGoqisOudF/FesZt9FZ2ZNeIdxlyaRvY+W3QGb4SuAxUalaViQz82zz1LTnoqevXPROqOUNU5DrwioWwb2DGVFd4TmHKtOt5OGhb3qY7nvtukX0wj0VHDuMIsYk9lUMHXkTfalqN9ZV9877GRxsFbB/ns+GdcSrtEuFs4H9b/kEjvyD9tgzU/n8TpM8hYsQJDeDi+n3yCPiQYAIvFws2bN7l+/TrR0dHExsbi4eFBr169cHe//36q0qPzIGfuZmC8oignhBAOwHEhxNai1z5XFOWT3x8shAgHegMVAF9gmxCijKIolodZcUl6WiiKwo5Jg/Fdd5jtVd1Z/PwEXjo9nYzDdmgMIWgdOtB0QCUc3W1Y/8UpLKZb2FlW0sL5KD6GTKg1AvyqYV79Iu/bT2ZRTDj1S7vxUcUAlCWXyS20sFhnYnFWFj1qBLCwQTClPe89rn057TKfn/ic/Tf342fv98DLBxRcusTN8RMwRkXhOnQIrqNGkZqVRfTBg0RHRxMTE4PRaATAx8eHhg0bUr9+fXRybfVi85fhrijKbeB20eNsIcRF4P6bJUInYLmiKIVAtBDiGlATOPgQ6itJTxWrxcL2Cf3w//U0m2p6sW7AcEYd+5rUE3ao9WVxD+pK2xcjSE/IY+3nJ1GLizjmrqST33n0eg10Xgw6OzKXDmG0ehp7UwMZXDOQF7JVmNdGcV0Hb1pz8PB2ZG3H6lTyv/d0xbSCNL44/gVrr63FXmfPhMgJ9CnX5y/vMs3Pz+fq999z45dfyPHypLBNazIsFjI+/hhFUQBwdXWlcuXKhISEEBQUhK3tP1u/XXq4/taYuxAiCKgKHAbqAaOFEM8Bx7hzdp/OneA/9Lu3xXOPXwZCiOHAcECuyyyVSOacHPaN6IH/sRjW1vVnZ9+eDD28lJQzetT6SvhX6EL70VW5ejSRPcsvolHvx9e4idYBVxHOAaj6/QR5qUQtHsHzpg+It7rxQcuyNNyfTEGeibkUsEMneK1zJTpH+KG6z3Z3/7nDNK0gjQHhAxheefhfzlm3WCzsX72avefOYVKpoFo1NGo1bmo1vl5eVK5cGTc3NwIDA3F2dn4EvSf9Ww8c7kIIe2A18LKiKFlCiDnAVO6Mw08FPgWGPOjnKYoyD5gHEBkZqfydSkvSk64wNpZTQ3rjcSuDBW3LEt26IQMObCHpPKj1kYTW6EyrYRU5/msMx3+9gEr5lSqavdT2i0MpVR/R6wfIjGf34ncYnT8FnY0DP3SqjPe6GNLzTIxX5dGkQSm2Nwu7a0rjfyiKwsrLK5l+dDpetl782PbHv7zDVLFaubJ2LZuPHCHdYMAnLY2aEREEdeuGk5OTvOHoKfJA4S6E0HIn2H9UFGUNgKIoib97fT6wsejpTeD3S7n5F5VJ0jMhe98+osaMQliNTBlcC8cqwXTdf5LES/loDPUo36gDjfuXY8/SK1zcdwK1aQMtPE4QZp8EVQcg2n2Gkh7Lt/NnMi1vNGU9bZnTtRqmxRfJzTfzmavC7EH1Ke1pf9865JvzmXpwKhuub6CBXwM+bPDhn56tW41GEtesYfuePVzz9MRWCNr6+FD9tddQ29x9QVZ68j3IbBkBLAQuKory2e/KfYrG4wG6AOeKHq8HlgohPuPOBdUw4MhDrbUkPYEURSHlu29J+vgTEt3h3UGtiPQVRO6OJTEqHY1NU6q360CtDsFsmX+ea8d2YzBtomvQJdw1GdDyfag1EuOxRUzZeI0Vpq60CrPn/fbVSJx9Bk2hhRXBBr4aVO2+Z+sAN7Ju8MquV7iafpWRESN5ofIL971gqlitpK9cxZGf13AiJASThwfVvb1p+dxz6OXY+VPtQc7c6wEDgLNCiFNFZW8AfYQQEdwZlokBXgBQFOW8EGIlcIE7M21GyZkyUklnLSjg1pQpZG/YyJGygk/6dqGbTRKBmwtJTshAa9uaer3aU6GBL+u/OkncuV9wYyfdSl/FRqsgui8Dt9KkLuzOi9H1OKLU56U6bvStGU7M16dwMikcrO7KO90q3HdsHWDnjZ28ue9NVCoVs5vPpr5f/fsea7p5k5gpU9iu0ZJQvjx+zs506N0bb+8HX6NdenKJ/1zxLk6RkZHKsWPHirsakvSPmNPTuTFsGIXnzrO8oYof2/ZhuPEStr+ZMOaZ0dq1o9mg1pSq5Ma6mUdJuvoTYYYDtPaLQuXkjej5A0Rt5/LOHxla+ArJwo2Pe1Ql2N5Axrfn8LUK4pr50bjF/ddjMVlMfHnySxadX0R51/J83uRz/OzvPalNURQyVq4iZuZMdtWuRZajI23atiUyMlKOqT9lhBDHFUW55w0K8g5VSfoXTIlJxA4ZTN6NGD7vpmJfrf6MTTmJebsRo2KLvUcv2oxsjKObDaum7ybjxjLqux4h0i0epVRdRP1xsG4U22/rGWN5BztbW1YMrElcQjYJC85SGhXGTiE0ruN/3zrEZccxcfdEzqWeo1fZXrxa41X0av2963vrFrcnTyH+wgX2NW+G2caGfr16Ubr039vUWnryyXCXpH/IdPMmMYMGkZt0iw96Cs5W6cf46GPkHjAi1F54hfWlw0t1MBZYWDltE6aUH+nid5JStmlQ43mEwRllSQ/ma3rzoakDFfwcmdc/km1brhF0MpVSqLHpEUZw9fsPk/wa/SvvHnwXlVD96ZowiqKQuXo1iR9O55a7OwfatMZgb89z/frJYZgSSoa7JP0DxpgYYgYNIjszhfd7qYiq2IdXTh4j+0IhKm0ZyjcYQLPBlUmKyWL95z9jk7uMnkHncNSZoNVHELWDgiOLmewwnZ9SAmhXyYd3moRx4puTNM20kKXT4Nq7PA7hbvf8/jxTHjOOzmDN1TVEeEQwo+EMfO1973ls7sGDJH3xBQWnzxDXrBkHPT3w9PSkX79+ODr++Toy0tNLhrsk/U0FV64QO3gwWQWZvNtHRVLpHry46yTZNwvRGGpSv+8AqrUM4tLBW2yZt4ggzQbaBl9Fbe+KaP0hmTu+YElKKItU35KcomZsk9L0sGrJ/OoUpRWFq6UdafRcRVS6e/94Xkm/wqu7XyU6M5phlYYxMmIkGtXdx+adPEnyFzPJO3wYtY8P118azdHkZEqHhtKjRw/0+nsP3Uglgwx3Sfob8s+dJ3boENKtubzTV4PJrz3PbTlJXroZG+c2tB/bm4ByruxdcZILm2bS3PUAFZyTsPpWIyF8KAtW7We5cRx56GkQ5s7Hpb0IPpgIGUZOCDNeXUrTpFbAfb//aMJRRm0fhZ3Wjnkt51Hbp/ZdxxRcvEjyFzPJ2b0blbs7OS+P5biikJCYSPXq1Wnbtq1cevcZIMNdkh5Q3okTxA4bToq2kLf7a3Fwbk67385iKtRi49qD3m91xdHdwJoZK3G78QmDS0Wh1QiSK49iemwZ1m50RNCEDhXceb5JBXyPJZP7axzxWPnBzsro56tT3uf+wyQHbh5gzM4xBDgEMK/FPDxsPf7wujklhYRp08j+dTPCyYmsF1/khFZDYkICLi4udO7cmSpVqty1pZ5UMslwl6QHkHvoEDdGvMhtOzPv9NERpKpN1S2XURlcEXbtaTG0ATqD4LfXXqSxfj1uXvkY/evzo+cYphwwY08eQ7xjGNx/AN4ODiQvuUju1QyWUsjRAAOzn6uNh8P9h0n2xO/h5Z0vE+IUwryW83A1uP6xfkeOcHP8eCxZ2WQ8P5STBgNJKSm4urrSuXNnKlWqJM/WnzEy3CXpL+Ts3Uvc6NHEO1p5p7eBajkVCToRi4t/GDnZzancuDQ21ihufzCcdo5x5AgXCrvO5Z2Lfiw7kEhn9X7ebRWIU4OXsGQbif3qJKqUfD6lALua3vzQoQIG7f2Dd3vsdibsmUBZl7LMbTH3D8sIKFYrqfMXkPjll9yKiOByZHVSMjNx0+vp0qULFStWlKH+jJLhLkl/Inv7duJffploN4WpvexpfDMI96vJlK5Rn6RbNfEoZY+fwxk8NoxAZWMl3qc3rgM+5cVlZ9hxNZGRmvW82qsVolJXsuOyuDX/LMJoYaadhd69q9IgzONPv39zzGYm7ZlEBfcKfNP8Gxx0/79Ouzk9nRuvvca5W7e50q0rOSoVHjodXbt2pWLFivKGpGecDHdJuo+szZuJHz+BKC/4oLsjra/4YH8rh1pde5MQUxbFmke50kn4H3mRfIuWvM4rMYTXos+CQ5y9lcVU7Q8M6DsAynfg1J5YbH+NJU9R2FXJkRndK/3p+jAAG6I2MHn/ZCI8IpjdfDZ2Wrv/vpZ++Ai7Zn3NJV9fCn198ff3p339+pQpU0aGugTIcJeke8pct46br7/OZT8VM7q50uGcGzapRlqNfIXMlFIkRt8gvLKZMucGY7SqyO64BHNIdXrP2ktSRg5z9bNo0XccBSEtWT3/OPWicrmlBnXPMrxSxecvv3/1ldW8e/BdanrX5MumX2KrvbOIV2FuLtu++YZTaWmYQkII9vKiUZs2lCpVSl4olf5Ahrsk/Y/0lSu5/fY7nCul4rPOHnQ+7Yg+y0r7V15HayjDrmVn8PSyUvPWAMwKJDX6iizf6gydtQ9RmMkyw+dU7TuV7IBGLJx5iO4pVuKdtFQZVRU7R8OffreiKHxz5htmn5pNPb96fNH4CwwaA1arlZMrVrD9zBny9HqCrVaa9euHf1jYY+oV6Wkjw12Sfid9+QoS3nmHk6EqZrX3ptsJW7R5KjpPnIJbQDgr3j+CwWClpXkoCAtRFd/iimtdpsw7iLeSyiLDxwT3/5Jkjzp8MfMQw9MVMnztqDWqKkL958MlZquZ9w+9z+qrq+kY2pF36r6DVqXl1v79/LJ2LTft7HCxWOhYtSplO3aUZ+rSn5LhLklFsjZv5va773IyVMXs9gF0PaZFZ9HSdfJ7WK3erP3sBMZ8E52cJ6EVORzxHMGvusas+OkMdfXX+UrzFW4DviXOsRpTvzrApCw1Rg8bwkdE/GWw55nyeHXPq+yJ38PwysMZHTGa/Ohofp07j1M6LSq9noZeXjQaOhS13HRaegAy3CUJyD1wgLgJE7jiB/PblqLrETUGlS1txr7F2V1Wrp86jVploYXjJ7hpotmg7sMiWnP+aByj3I4zLv9r1M+t5aKuIm9+fZAP8rSonPWUGhGBSvfnUxFT81MZvX00F9IuMKX2FLoHduDEjBnsTEkhx96eMJ2O9kOH4uTl9Zh6QyoJZLhLz7z8s+eIHvkiN12szG0XQofDYGvjSHij0WxZmIzVYsVJfYtObu+gI50F+T2YZ9cBa1oeC8odp3nMp9B1PkeVckz85hCfGw3Y2mnxHV4FtZ32T7/7RtYNRmwbQXJeMl80/oKIG1qWf/YKV3y8cbKzo2+HDpSpXv0x9YRUkshwl55phdejuTJ0IBl6I7M7hNLiuIKNrSsGpz6c25OPwEJtpxVUs1lFmtmB93OfY4W+OeEutnwTfp7A/Z9Cgwls1zbitfmHmYUd7lo1nkMroXH984unp5JOMWbHGAAW1J2JZf6vLMjPJ9fbi8jgYFr17YtW++e/HCTpfmS4S88sU2Ii557rg8mcx+zupWl42opO54KV7uRnG3DRxNHZ5xNsTHEcywxisuoFLulL0aO6P1PDb2JYNQlzuY5My+nM0sXHmKtzxM8icB9YAZ3v/TevBvj56s9MPTQVbztvvlD14+TUBVwMDMDRwYGBffoQXKbMY+oFqaSS4S49kywZGRzv1x1tVibfdAuj5jkLWp0LQtsLobKhScDPlDMtwaRyZVlcVRa4DyZauDG1UwUGBOfAt8PIca1A59i+3Ey/wbcubpTKMOHapyyGUOf7fq/JauLjox+z7NIyWhiq0WePMxvFMbJKBRIREkKbXr3kUrzSQyHDXXrmWPPy2Ne/Ky63U/imUxiVLlnQ6l1Q6XpjUJvpGzIZ29yLJDjUYOlxA78E9yPG6sjH3avQvawO67x2ZFsNtLw1gnLONsx3tkGbYcS5Yyi2le+/nEB6QTrjd48n+vIRpl8qQ25qIZvKOmGr0dCvZ0/CypV7jL0glXQy3KVniiUnl119OuAddZtF7cIIvW5BY3BBpeuDvSaPfr6vorEKjtp2ZdvRDLaV60+M0YbPe1WhUwU3Mr9phT4riQGFbzGxdAVqxeShsgHX5yv96Rn7pbRLvPfTKOrvSGTQLVeORgaRXd6RSmFhtO3aFRsbm8fXCdIzQYa79MwwZ2ayu08HvKKTWdK2LN7xJjQGN9T6Pjhp0ujtNRG1jSO/ZNTn9LVUdlYcQmyemq/6VKV1WWfOf92TCpmn+NDwJp/7NkR/LQt9GRdce5ZBbX//uefb9v1A1JcfMfESnKscwY6mpXG0t6d/585yY2rpkZHhLj0TzGnp7OnVDs+b6SxtXwH3uAI0Bnc0hj64am7Tw30SVoMrS66W52ZeNjsrDyM2R2F2v2q0KKXmxswWVMg9y173txiY0wBrTDZObYOxr++HUN37TtHcgmw2vTuU8mvPovj5sLVrA3IR1KxZk2bNmsmxdemRkuEulXiFyUkc6NUBt8QslrWvgltsDhobTzQ2PfHS3qCL6+vkqFxYeioAAkPZEdqKG5km5g2IpLFbBhlfdcErP5GDLvMIvuWLcAKPEZXRB95/16QT57cT/+oEysRZ2NOuGUn27ri7u9OrY0cCAwMfY+ulZ9VfhrsQIgD4HvACFGCeoigzhRCuwAogCIgBeiqKki7uLHgxE2gL5AGDFEU58WiqL0l/ruD2LQ736oRLWg6rOlTDLToTrZ0Han0vAgyXae/4NkmFjvwcXxaHNoOZFe9CepaRhQMjaaC9TMHcPpiNes7Y/EhAoi02Fd1w6VYGlc29f3SMFiMrFk2kzNe/4WTrzJZebcizKDSoX5+GDRvKeevSY/MgZ+5mYLyiKCeEEA7AcSHEVmAQsF1RlOlCiEnAJOA1oA0QVvSnFjCn6G9Jeqzy4uM41rsTjpn5rOkQicv1dPROziD6EmhzlvYO7xKX58RBbUeUPoN5d38CXo6CVS/UpVLKJizrRnPTXJ5c9fv4ZqlxbB2EQyP/+y7YdTHxLHsnj6DB3jTOVwvnUvkIDAYbBvXoQalSpR5v46Vn3l+Gu6Iot4HbRY+zhRAXAT+gE9C46LDFwC7uhHsn4HtFURTgkBDCWQjhU/Q5kvRYZMdc51SfbtjmFbCufU2crqdi62GLxTSQQN0ROjh+xPUcN27W/ZC9BaXZuOc2Tct58lmPyjgf+RR2z+CUqSOODMNNrcF1UDlsyrre87tMVhPLtn2O64eLqJsoONS9A7EaW4ICAujWrRsODg73fJ8kPUp/a8xdCBEEVAUOA16/C+wE7gzbwJ3gj/vd2+KLymS4S49FxrUrnO/XA32hkU3t6uAQlYSDvwpjzjBK6U/QzvljzhnLUzjgO97Zlkh0SgKvtirLi+ULUa3uiRK1i72mcQRZm6L1sMF9YAW07veeqng6+TTLvn2VHkvjKLB3ZPegriTn5VOvXj2aNm0q9y+Vis0Dh7sQwh5YDbysKErW7/9rqiiKIoRQ/s4XCyGGA8MBeYFJemhSL5/nSv8+qM0mtrati11UIk5BZoyZY/HWRVHX7ku2W9uS2XEqb62+iq1OzZJ+5agb/RXMXUyh2oV9xq8pqwShK+eKe5+yqO6xHV62MZuZx7+gYPFyBuy2El+tMicrRqBYrPTu3Zty8oYkqZg9ULgLIbTcCfYfFUVZU1Sc+J/hFiGED5BUVH4TCPjd2/2Lyv5AUZR5wDyAyMjIv/WLQZLuJensKaIHDUDBzM6WdTBEJeIZkoMx4yXsVJkEiQXstu3P+cr9Wbj2EjVKOfN1mZN4bRiCYszlpHtfkuK6UBE9hsb+uLUMumuao6IobLuxjU/2fkCPNUlUj9JyoksbonR6vJyd6dWrF66u9x6+kaTH6UFmywhgIXBRUZTPfvfSemAgML3o73W/Kx8thFjOnQupmXK8XXrUbp88SvyQwRRqLBxsUht9TBKhQSkUZA0lHwP2hUu4HtKDTa6N2XXwBgPLKUzOGI1273XMoS35OLcfdWKcKS/UOPYsg2PVu9dOT8hNYNqhaZy5sJPJ67QU6IPZ3LUGJiFoWL8+DRo0kLNhpCfGg5y51wMGAGeFEKeKyt7gTqivFEIMBWKBnkWvbeLONMhr3JkKOfhhVliS/lf8kf0kDB9OjkHhWP3aaGKTKR96G3K7c9MSDHmrMdVswaLCcGKupTIt5Dz9YqaBR3mSOy9j/DZ7RqcoeKrVeA6qgE2Yy13fcT3zOkM2D8E3Opv3t3lwOrwSKa6ulAoMpH379nh43H9NGUkqDg8yW2YfcL/NGpvd43gFGPUv6yVJDyR6307SR44mw07hVO0aqOKSKVc+CXNaX+KMVTHlbcdQty6fJHoBhXzvv566t5ZB/XEcDXmRGd+f4e0CcDRo8RlaCV3A3TNbojOjGbp5CPWOFVAprgw76oZhsLGhU+vWREREyL1MpSeSvENVempdWfoD+R98SLKTinPVqyFup1CmUgY5CcNJNZfClPsbOfVq8vUNG4JcBAu0nxKUchw6z2GlqQErF5zkQ8UGg4MO72GV0Xra3vUdMZkxjF47mAG/FpDhXYdL5ZyoEh5Oy3btsLOzK4ZWS9KDkeEuPXUUk4nzU99BvXINF4M03AqtjEhOI6RKHmlxYyhQHDHm/MzZ8PpsjzHQOFDDl1ljcbTkktljFW+ddib91AU+Ebbo3G3wfL4SGqe713mJzYrl/fkDGLupkNPVG2G0t6d/nz6UlhtpSE8BGe7SU8WclsaF0SPRnjjNxkgdGqcKiIwMSkUIkmNeQa2yYs5bze7AOpzMcmJIOTNv3hiCytmXrVXnM2llLh0LUpmALTp/B9wHVbjnPqex6dGsmNyLQcdV7GvaDGHvwKCBA/Hz8yuGVkvS3yfDXXpqFFy4wNUXX0RJSWJOGxtCCsqiZGbgV8mRxOjBOGhSsbCDX9yqc8HqzbiwJMbEvEyhX23Gi1c5+ksmn+kdCFXAJsIDl65hqHR332QUe+0kp14aRJ0sB3a3bIqNoyMDBg6UF02lp4qquCsgSQ8ic+MvRPXpQ0Z+Ch/0ciA0rwzkZOJZPpDkmOF46qJxcD7AD6oILmgDea18GmPiXibKtz214kZjjVZYrnGkNGpce5XFrXe5ewZ79MaVJPXsj8Hkyq6WzXFwd2fI88/LYJeeOvLMXXqiKYpCyqzZpHz9NZcCtCxo7UjriwEoxiwcfWuTEVefEMMhHEOymHy1HNF2wUyOtDD03EscMtRn6PXeTHdwpWq2FV0pB1x7lUXjarjre6y5uVx+93VYv5UrlUoRVbEOnl5e9O/fH3v7P9/sWpKeRDLcpSeWoigkf/opqQsWsqOynp8a2tP5pA+KKReDU0cKMkOp4/gjthGhjDgUQIxdEO82sOe5kz25oi3Hp1kvss7WDUOOGcfmgTg0CUSo7562mH/6NFHjxsDNJDa1rkiecwUCAwLo27cvBsPdvwgk6Wkgw116IimKQuIHH5D+wxI2V9Ozsa4jnY97oFgK0dj0R2fR0dLrY2g6hL5rMrhhG8D7Lbzod7wniYoH63Om8JlwRGPQ4DqwAvpSd2+soZjNpMydS/Ks2aQ4wtYBTdCaPClbtizdunVDp7v/1nmS9KST4S49cRSrlYR33yVjxUo21tSztaYLHY66YLUoaGwG4aePp4X/ci5UeZWx6zK4bfBnWit/+p4bQla+CxfzP6QPNthGeOLcKRSV4e5/5sYbN7g18TXyT51id2Ub4iJbossz0LBhQxo3boxKJS9HSU83Ge7SE0WxWLg1eTJZP69lTV09+6u60uaQI4qiRWvbh0jH36geep73xVh+2KFBr3Xmk3al6Hr1VbKTSpNsepFSGg2uPcpiW+XeF0Gzd+zg5oRXMWJmfmd3XBybYGfS06VnF8LDwx9ziyXp0ZDhLj0xFLOZ+NcmkfPLL6xooOdEJXeaH7JFUeywdWxPG5eZ2AUYaJowmjijHTU0iXw5phOeOyeTHtWYfGs9Mpw0VBlRFa3LvcfK05ctI2Hq+yQHODC7uQ/l8iNxsXOhd+/eeHndvViYJD2tZLhLTwTFaCR2/ATyt25lSWMdF8t70vigDlRO2Du1obvb+1xyLMvAGwNxLMxgvF8yL44Zhvnnr0k81QoTzhzyM9BjZHVU6ruHVBRFIfnzL0idN4/oSp6srBlAudzShISE0L17d2xt7156QJKeZjLcpWJnLSgg5qUxFO7dy6LmOqJDPKl3UAtqdxwcWtHD7T1WqarzaVp3aqUe5vkmZWnU8wXyf/yStItVySWLZcG2TH7+PsFuNHJ7yhQy163nTJNyHAgIJCjXhTp16tC8eXO5W5JUIslwl4qVJSeH6yNexHj8GPPb6Ej0c6PWER0qtTeODs3o6v4OnyutOGSsRt+45XQa9BxVmrYge97HZMbWIY0kZgYFMHtodTT3CHZLTg43x4wh58BB9nRvxE2tB25o6dGrB+XLly+GFkvS4yHDXSo25vR0ooYOw3jpIl930JLr5kzkURtUGj+cHRvR0f093rD0IqvQmTbJG+gwbiKhlSqRNnM6eUmNuc0tttepwJx24eg0dwe7KTGJuOHDybh5k+39OpNr0aP30DB2wFi5abVU4slwl4qFKTGJa4MHY7wRy6ddVahtnal6wgGVJgA3pzq0dPuQ0dZhuOZkUifnMF0nv4+PvzcJH3+EObsZMapbFPZpyKRKPvf8/PyzZ4kfM5ZYWxsOdmxLodWCLlzFG93fkNMcpWeCDHfpsTPGxXFt4GAKUpOY0RMcVc6UO+2IShuEu2Mk9d0+Z5R6LGWSzxFoTaX729Oxd9Bzc8aXUNiMGEMiwWM64Od690VQRVFIX7aMWzM+4mTt2kT5eJOmScOhugOTW05GJWSwS88GGe7SY1V47RrXBg8hNyeTD3pb8TW5EHrWCZW2NF5OlajoMpeJNhOJiNqCt42g++SPyM7OoPCzH8HclAT3DGqO7YJOe/dFUGtuLrffepvkrVs50L4dSTodF50u4lfFj3cbvyuDXXqmyHCXHpv88+e5PmQoGdZCpvW1EJbrQuBFJ1Tasvg7h+DhtJRPXSdS4+RyPDxc6T55KilXz6NfdwqrtRF5pY1UH9r+ntvaFV67RvzYl0lLSmJ71w7kCRVHPA4RUiaEGY1moFHJf+rSs0X+i5cei7wTJ4gZNpxkrcL7fUxEZLjic9kRtS6cUs6+pDscYK/feCrvm4dXqSC6vv4uydu2oj+ixUollFo6ynRpcM/PTl23loS33ua2pzO72jXDorKQWi6JF6q+QPuQ9mjVd2/GIUklnQx36ZHLPXCAGyNHcdtezdSehdRJcsP9qgNqXWVCnJ055BCPJqA7gbvn4F++Ap1emUzKorWobwVgEsmoewYSULX0XZ9bcP06B6dPwHvPRY5U8SM6vA46Gy1denWhUqlKxdBSSXpyyHCXHqnsHTuIH/sy8W4G3u2RR4Ob3rhHGVDrqxLibGCVg5oy9kHY7lpImToNaN59CKmf/QYFpUjXnMZ1ZB/8fN3++3mKopB//Dip335L9s6duKhge4dapNgFEeDrR9++feXG1ZKEDHfpEcrc+Au3XnuNG372TO2YS+OrgbjcVqHW16CUi4pvnAKom3cR26jzNOg/hHC3yqTOOo1QbDlju5fIsa/g6WQD3Fl3JnvbNlK//Y6CM2cotNexrq4afc0eFNxWKF++PF27dkWrlUMwkgQy3KVHJH3VKhLeepvYEGdmtMijxakQ7LOtaGyb4u6cz2zHsjSK/wVXUUD7N97D4Vwh6ftvoxXXWOGcQ7/R43Gz1wOQ9dsWkj7+GFN8PNrAQA73i2Chx3Va5ren4LaFunXr0rx5czl/XZJ+R4a79NCl/bCExGnTiA534+vaBbQ8Vgq9WYPOvi0GlxiW2FWh8ZUfKRUUSIeRUzGuPEFusjMq9UY+dQvjzRFjcbLVolgsJH8xk9T58zGEh+P+5Rd8qN3GmQvnaJ7SFq1GS4/ePShXrlxxN1mSnjh/eaojhPhWCJEkhDj3u7J3hBA3hRCniv60/d1rrwshrgkhLgshWj2qiktPpuwdO0n84AOuVXZjcSULTY/7oLc64ezUBov7RXbYBNHk0vdUb9CQ7oOHUzD/IIXJDsSrNzDFvwlTRg7EyVaLJSuLuBdfJHX+fJx79cJv6Q+8p/6NhONJ1Equhb+PPyNGjJDBLkn38SBn7ouAr4Hv/6f8c0VRPvl9gRAiHOgNVAB8gW1CiDKKolgeQl2lJ1zh9evcnPgqt/1s+cVfQ81Lzqi0wZRyDuSC4yUK7EKpfnYtzYa8QHlVNqnfXUNRHFgljnKxTA9m9a2GQau+M2d91GiMt27h/c472PXowsRNExFnBMGmYOrXr0+TJk3kao6S9Cf+MtwVRdkjhAh6wM/rBCxXFKUQiBZCXANqAgf/eRWlp4ElO5sbI0eSqjazI8ibsJsa1PpIIt1S2O6STbB3RZK3/0TjXr0pG32AlJgmWFX5jFdSCa3RitmdKqJRq8jeto1bE19D2NpSavEizBVLM375eJyuOaHX6enTvw+lS989LVKSpD/6N1egRgshzhQN27gUlfkBcb87Jr6oTCrBFKuV2AnjiE9PZl+oP87ZBvR2zWjtc4QjoWVpEhFJ8vafqFq3BmFnD5Aa04osbT6drCpqNa3OB10qoRaQ/NXXxI9+CV1oKME/rSKltDsTvp2A21U3nD2deXn0yzLYJekB/dNwnwOEAhHAbeDTv/sBQojhQohjQohjycnJ/7Aa0pPg+qczOHn1IieCfNDijotzKzr5/MCZmgNpXzOSo0vnEVY6kIgkC1n5fbhqKKCLSc2YDhUY37IsxuvXiR3wHCmzZuHUuTOllvzAWeUW7y98H98EX/zK+PHysJflMr2S9Df8o9kyiqIk/uexEGI+sLHo6U0g4HeH+heV3esz5gHzACIjI5V/Ug+p+J1buoCDe3eQ5eGMWh9BKWcHqjt9RVzr2TTw8mHlu68T4O5KLWtl8pQK/GZbyIwCC5/2qUr78u4kz55N6pxvELa2+HzwAU5dOvPLlV/YtHYTvvm+VKlVhc6tO99zPRlJku7vH4W7EMJHUZTbRU+7AP+ZSbMeWCqE+Iw7F1TDgCP/upbSE2n3D3M4sX4jio09Wrs21HA5hY3tJi63X0qNIG+WvjkOb4Mz9e3bYcKNr+yMbDCZWTi4BtVz4onu9gKFV6/h2LYtXm+8jtrNjW+OfMPl7ZfxMHnQvE1z6teqX9zNlKSn0l+GuxBiGdAYcBdCxANvA42FEBGAAsQALwAoinJeCLESuACYgVFypkzJtHbWh0Tt2Y9a44PGsQ1tXBZwVWsivu0y2of7sOyNl/HGk5pu7bDqVYxXWbiiWFjarxLey+cSu3QpGm9v/L+Zg0PjxpgsJiZvnozpmAknnOjTpw/lyshpjpL0Tz3IbJk+9yhe+CfHTwOm/ZtKSU+2X7/97E6w6yqht69KD9f3WCUqoWn+Dn3KubLm3Qn4GUOp4N4Qi7OZAXlGTDoNKxu6wYgBpCck4NK/Px5jx6K2t8OqWHlr/Vuozqiw19szbOAwfHzuvcOSJEkPRt6hKv0t27+cyoV9h1HpQnG2C6S7x3jetfTGu+Hz9PIsYOmrL1HJ0IAAlwrkBKjofqsAT1cbfqhgoeCl4agcHQlathSbiAjgzkJgH638CN1FHTonHaOHjMbJyal4GylJJYAMd+mBWI1G9r02mtM3ExEaD3xdXGnj9h7DjK8QVqstjU2XWPf+Uhp5dcJZF8Ct8nr6XEqhop8js91ukTX+PfSlSxMwby5aLy8AjEYjXy35ioIbBai8Vbw65FV0Ol0xt1SSSgYZ7tJfMsbGcmL0KE7Y2KCoDIR52dLQZS7d89+mfKUaREZv4vixc7T27Ytebc/FChaGnU+mXqgrH+UdI+vd2djVrYPfl1+itrcHICMjg/nfzycnLYf8oHw+GPABGrX85yhJD4v8aZL+VNavv3L5nQ85HByMRcmggq+Bmq4b6Zz7HiEh5ah8/FsyUxVa+/dHIwr5xTeVaeftaB/uzqSL68j6+WecOnXCZ+p7iKKz8piYGJYuX0quMZekMkl82fNLGeyS9JDJnyjpnhRFIeWrr4hetJYD5SOwmK9T2ctEZbeLtM9+j/BAPyrt/wpPu3AivBuhFTeZb5/C4tthTGkaQPOVX5K9bx/uI1/E/aWXEEKgKApHjx7l182/kqPJ4XrYdRZ0WYCNxqa4mytJJY4Md+kuiqKQ/OmnXP1pH8cqNsZSeIRwt2zCvdNon/U2kf4ulN7zJdWD2hNEefSqU8wSN9mqbsmqjh44TX+LvKtX8Z76Hi49egCQkpLCL7/8QnR0NOkO6Zz1OcvidotxMbj8RW0kSfonZLhLf6AoCokfTufc1uucL98Ec95Wgh2yKOufT9usyTT00hC89xualumPu8kHW/UWlnCL+DIvstojlcyXhmJSqQj4Zg72DRpgMpnYt28f+/btQ6VWcSvwFqd0p/i25bcEOAb8dYUkSfpHZLhL/6VYrdx8byrHj5mJKVUOc95W/GyzKBtspWPWRJo75xNyZCUty43AvtABR833/CaSsG36MdOOrSftw4UYKlTAb+ZMdP5+REVF8csvv5CWlka+Rz7bDdvR2ej4tMGnVHSvWNzNlaQSTYa7BNwJ9pjJ73LwqhvJbjmY83cSap9KQLCBbtkv09YmgdJn9tK67IvoCrS4aD/lmDaPsA5f4vLZ+6QdPYpz7154vf46eSYTG1av5uzZs5j0Jg55HcLsamZE+Ah6l+2Nvc6+uJsrSSWeDHcJxWLh0mvvcTAhjCybK1gKThDulIhLKWf65YyknbhK+LVLNA8djqbQhKt2Cou04bSp2wfjmGHkZ+fgO2M6Tp06cf36dZatXIax0Mgl50uk+aQxuNJgupXpJi+cStJjJMP9GaeYTJyY8CHHsquSJ/ZhLbxEhMsttP5eDM59kfaFx6mWmUGDwOdQmzPQ6z5irOjCK+6u5I8dhS4ggMCFC9GHhbF37162b99OljaL6NLR9K3Rl06hndCp5Y1JkvS4yXB/hllzczkwZjqnVXUpNG5AMd2gltsN8n0CeSH/BTrkHKCuSUOkR1c01jiybL6nj/FFZtqmwJefYN+sGb4zpmPSaFi+YjmXL10m3i4ep6pOLG+yHL1aX9xNlKRnlgz3Z5Q5JYUDoz/kjGNzCvN+AlMiDT2vk+hZjteMQ+mcuYv6iiNVnBqjV53mjPMpRqaN5HvVWfTfL8WxYwd8P/iA5LQ0lq9YTmpqKmddz1Kvbj1eqf4KKvFvNvmSJOnfkuH+DCq8Hs3BVz7hnFd78vNWIEwptPS+ylW3KrxtGkiXtG1Ut9pQxakhBvUxNgeYmRLdicX5B3DcshbnXr3wfvstLly8yM9rf6ZAKeCAzwFGNB5Bz7I9i7t5kiQhw/2Zk3fiJIfemMOFwM7k5S9DbUqjje9lTrrU5CNLf7olb6ICWmo4NUeniWJ9hQDeP6VnQfJWPA5sxXXgQNxencCWrVs5ePAgWTZZHPU+yrRm02jo37C4mydJUhEZ7s+QrK1bOTJjJZdCioLdmE5b30vsc2rIHGsPuiaspzSCui7t0WhSWF7Wm5knzcy+sR7fU3eWEtAPGsT333/PjRs3uOF0gxu+N5jXfB7l3coXd/MkSfodGe7PiLQff+T4wt1cLt2F3Pwf0Zgy6OB3kZ9tW/KLvhtdo1cRiIXGrp0QaiNv2BnYe9bE/Otr8Dl3BI/x48hp0YJFc+dSUFjASa+TqHxVLGm+BG877+JuniRJ/0OG+zMgdeG3HF96hCulO5GbvwSNKZNOfhdYZOhEjFc7Op1ajJtQaObeCaswMMKajzq7gDUXl6O5egnPN9/kUkgw2xYvxtbJlp1uO3F0dWRhq4W4GlyLu3mSJN2DDPcSLnXBAo6uOM210Pbk5P+AzpRNR/8LzNb3QuVSkYh9M/F2tqWJSxvMuDNayaepNoF2vy5ECHD74nO2ZWZycetW/EL9WMQiXO1dWdBqgQx2SXqCyXAvwZLnzufomktcD25ZFOw5dPS/wBz7QdjnmHE6upzIQEF5TT0KlBDeU+UxMW8/bmt/Rl+pEtopk1m6axfp6elUqV+Fj5M+xknvxPyW83G3cS/u5kmS9CdkuJdQyXPmcmhDNLGlGpKT9wN6cw4d/C+x0Ol5nK9dxF1jok1pLXprXwoVL9ba5PLq6YVYL5zH5bkBJLdpw/oNG9Dr9TTr1oxJ5yZhr7NnYauFcoxdkp4CMtxLoOTZcziw6TZx/rXJyfsBgyWXdgFX+dGmLy7nDhMW7EUDlQcFls4kYOaSIZ5Wv8wERcFn5hccBg6uW0dAQAC1Wtdi1L5R6NV6FrZciK+9b3E3T5KkByDDvYRJ/HoW+7ekcdOnKrl5P2BryaNlQBTLTa1wijpHkypBeGdXp8Aawm/k4qS+RMTyuegqVMD5ww9Ye+AAMTEx1KxZk9K1SjNi+wgEggUtF8j11yXpKSLDvQRJ+HoW+7Znc9srnNy8H7Cz5tE0MJbV2fXxtGbTsVxFLFlNyRNmJpNDf90FglYuwL5ZM5QJ4/l2zRry8/Pp0qULKS4pPPfbc/89Yw92Ci7u5kmS9DfIBUBKiJRFi9m7PZfb7qHk5S3BQcmlUWA8G1KqEKS20sm1LpbcVsTqsuiOhRf05+4Ee+vWJD43gMVLlqBWqxk8ZDB7rXt5eefLBDsGs6L9Ckq7lC7u5kmS9Df9ZbgLIb4VQiQJIc79rsxVCLFVCHG16G+XonIhhPhSCHFNCHFGCFHtUVZeuiNt9Rr2/3SNBPcg8nKX4qDkUjswid9ul6GcIZ/m9jUxW8vzk20qz1v0fGM4i8eKb1F37MjeGpFs/PVXgoKC6Du4Lx9e/JC5Z+bSKbQTi9oskhdPJekp9SBn7ouA1v9TNgnYrihKGLC96DlAGyCs6M9wYM7DqaZ0PxlbNnN47i5ueFegIGcpTiKHyMB0dsf5U9X2NnVdWmJUKvKx1sgixYYV2lPYLfuO2926sdrFmevR0bRs2ZI67eowdMdQ9t/czxu13mBqvalyyV5Jeor95Zi7oih7hBBB/1PcCWhc9HgxsAt4raj8e0VRFOCQEMJZCOGjKMrth1Zj6b8y9+/m2IfLuRbagsKc5Tipc6jon8X+GE/qu8fi5fkqxrwgppNPbrATK2K3kL1xLUd69SJeQClvbzp27Mip3FP039wfvVrP/JbzifSOLO6mSZL0L/3TC6pevwvsBMCr6LEfEPe74+KLyu4KdyHEcO6c3RMYGPgPq/HsyjxxmBOvf8XFsA4Yc1bhrMnBz9vE0VhXmgSlkqZ9H988F1a5CgZ2q0bAj3M4evwIZzp0QKXT0a5FC6pVr8bcM3OZc3oOFd0q8nmTz+UwjCSVEP96toyiKIoQQvkH75sHzAOIjIz82+9/lmVeOMGpMe9wLqwbxtyfcdbkYHDXc/WmGo9gF04p42luMpBUw4OxHUO5NmUKa7KySI6sTmhoKB06dEBjq+HlXS+zK24XHUM78ladt+QwjCSVIP803BP/M9wihPABkorKbwK/nwztX1QmPSSZF49zavh4zoX2wJi3ASddHkZHRwoSczH6+WDRDqa5UYVDq1L41vXiwGuvsUevR/HyomOHDlStWpXYrFjGbBrDjawbTKo5ib7l+iKEKO6mSZL0EP3TcF8PDASmF/297nflo4UQy4FaQKYcb384FIuFy3M+JHvhBi5U7EVBwRYcdIVk2rhgm55Bpl9pKrr0o1q2FYdmgdhUc2LV669z0dERD52eXsOH4e7uzp74Pby25zW0Ki3zW86nhneN4m6aJEmPwF+GuxBiGXcunroLIeKBt7kT6iuFEEOBWOA/e6ttAtoC14A8YPAjqPMzp/B6NKdeGYo+Oo9DVbtiLNiFrc5KqsYJt9xUsvzq0sGuFS45VhxblKIg2Mz3H3xIqqMjEe7utB8xArVazfwz8/nq5FeUcy3HF02+kEsJSFIJ9iCzZfrc56Vm9zhWAUb920pJdygWC3HffkPGzFmYbF04UL0t1ry9GPRaUhU9vuYUNH7daKWuhFanwXVAWS7ePscv3/2GSiXoVK4cVXv3ptBSyKQ9k/gt5jfaBrflnbrvYKOxKe7mSZL0CMnlB55QhdHRXJwwGv3565wo60m6S12seUfQ2tqSaFJTTpOFr/coSuOBoZwr9p2C+XXTWk5fuYJ7RgbdunTBp2lTso3ZjNkxhmOJxxhXfRyDKgyS4+uS9AyQ4f4ESt20kVuTJmFSWfiufVmCk0phzT+L1cGVOKOaRjaC0m7PY6fW49wuhMKyer799huSsrKocO0abV55Bfvq1UnJT2HE1hFEZUQxvcF02oW0K+6mSZL0mMhwf8Ikf7+Y5A+nE+UrWNShHY2PZWExRZHtHEBSoZpu9naUcWwHbjZ49Q/nemYca2athIICml6Ppta099GHhhKXFcfwrcNJLUhlVrNZ1PWrW9xNkyTpMZLh/oRQrFYSP/uU9AXfcixMsKllRxodvIXFks4t93CSjSp66Qso49gOTRlXPPqWZefuHew/eBCXtDRaWBXKzp+H2sGBi6kXGbFtBFbFysKWC6nkUam4mydJ0mMmw/0JoBiN3HzzTbI3bGRLVcGZaq2peSAGq7WAi541STWr6CfiqejSB22QE/ZdS7Hk+++IuXWLkKgoWtSqhffIkQiVisO3DzN251gcdY7MbTFXLtUrSc8oGe7FzJKTS/yYMeQdOMCyhipyfdoTfjwKARz0bkyGBfqYzlHNcyB6T1tMLV34Zs7X5OflUevsOeqPGYND0yYAbI7ezBv73qCUYym+af4NXnZef/7lkiSVWDLci5E5OZkbw18g7/Il5rbV4SY643vpNBq1ls3ejcmyaOmRf4xGfoOwsdWR3ljP6h++Q5+TQ+voaCp/9hn6kGDMVjNfnviS785/R1XPqnzV9Cuc9E7F3TxJkoqRDPdiYoyJIfb5YeQmJfBlFyfKprdDk3oEvVbHUq826Ez5tMo9SYegQdgoghu1zGxe+xPOaWm0MVsIXbQItb09aQVpTNw9kcMJh+lVthev1XgNrVpb3M2TJKmYyXAvBgUXLhA7bBiZBbl83TWAiLh6KDkHsdNr+NGzDRXSzhIssuhVdjj6bBPnIjI5sPsoPrdu0cbHl4ApkxEaDWeTz/LKrlfIKMzg/Xrv06l0p+JumiRJTwgZ7o9Z7uEj3HhxBKk6M9+2r0bV6GCsBQdxsBVsc61Ni1ubcXB2pkel0VjjsjkYGsf5s1cJjrpOm7p18BgxAiEEP135iQ8Of4CnrSc/tPmB8m7li7tpkiQ9QWS4P0YZW37j5vjx3HKysqRFI6pec8FiPIqLo0KUwZ+a8fvwLBVEq5DnyY1JZ7fPZWLjEqh4/jzN+vTFpWsXCi2FfHD4A9ZcXUM933pMbzAdZ4NzcTdNkqQnjAz3x+TS4q+xTJ/FNR9Y0rob9c6BxXgKTzcT6SYDrknxhJcvT1Wn3qTHpbDd9SwpaRnUOnWaBq++in39ehy+fZiph6YSmxXL8MrDGVllJGqVuribJknSE0iG+yNWaC5ky/sjKL38EOdDtXzXbhytDsdjKTyEn1seqQU2mHIt1I2oQhAdicm4yS7DGcyZBTQ6f47aH35AfogPb+57k/VR6wlwCGBei3nU8a1T3E2TJOkJJsP9EUrOS2b9K12puzuFS5G+zGnyBp32HsdScIgA5xySC+woyBU0r1Yd1/xWHBFXOCmicErJoPXt25T/+ms2Fxznk7UvkGPMYVilYQyvPByDxlDcTZMk6Qknw/0RiUq9yq6xfah7LJeE1vX4ovIQOu89iiV/DwGO2WQb1RTmQdtqtdBk12ez9hg3lTSCr1+nWVgZTG+OZOTJtzmScIQIjwjeqvMWYS5hxd0sSZKeEjLcH4Fj8Yc49/IL1D1nJL1Le94O7kzn/aex5m3D1y6HAotCXoGObo26kxzvzg7dQQrN+dS8cIEGI0eyL8TI5N/6oFfrmVJ7Ct3LdEclVMXdLEmSniIy3B+yzZc3kDJhErWuWknu15c3PJrQ5fBFlJxf8bTJxao2oRg96Vr7Jc7cvM0R7XHssnNol5pKxS+/ZGn6Vj7d8ynVvarzccOP8bD1KO4mSZL0FJLh/pAoisIPx+ZjmPwF1W4onB46nKWaSLoeu46SsxFXfR62hjxsVXWp6NeVfSkXuai9iV98PK0rVsTnw2l8cupzfrz4I62CWjGt/jT0an1xN0uSpKeUDPeHwGK18Nmu9yjz/kpKJwh+HjyOjLQwmiUcx5y3AydtPr7O2bjrR+CmDmY7J4hRpxEeE0vbF4ajqVGVifteZ2vsVgaED2BC5AQ5DCNJ0r8iw/1fup1zmw83TaTN18fwT1OxuOdbeMU445u9EbPxEoF26fi76PDUTkNRqdik2keSYqJ2VjbNP/uUXDs1I7cO52TSSV6NfJXnKjxX3E2SJKkEkOH+DymKwvorazk0dyoDt+dj1nmyptVrBNzMx5K3CIs5i3DXJFw1LfHTtibPMZ+NmdvJUWto6+VDjXfe4VbuLV789UXis+P5uNHHtA5qXdzNkiSphJDh/g+k5qcye8UEIhYf4rnbcCKiI8luLXHJuIgldzMalQlPh0Jctb3xs61Jmm0MmzLPYdVq6dO8OWGNGnEs4RgTdk/AaDUyt8VcanjXKO5mSZJUgshw/5t2XvyFC9On0PVIPnmO9izvNBGPDGcM+WvJzY3G1mAmU7GnjlNzvLQ1ics/zHZ1OnqDgcFDhuAZGMi3577lyxNf4u/gzxeNv6C0S+nibpYkSSWMDPcHlGPMYdms0VRYcpjGeXCmeWtOO3bEPzkPbcE8cgrzyXZ0Jc9cSCffGrhaanEucyeHPcy42Tny3IsvggHG7hzLzridtCjVgvfqvoe9zr64myZJUgkkw/0BRGVEsemN52i+I430IHfmdJ2I70UHfJMTUOUuIc9i5Yx7VWpwkebBTbDNqsL+3O1c9IIQvwB6DhhAdG4047aOIyE3gddqvEa/8v0QQhR30yRJKqH+VbgLIWKAbMACmBVFiRRCuAIrgCAgBuipKEr6v6tm8dl0fROHPnuDPjsKiW9Wh1nVX6DJsVxUyhms2ZtBWDjuWZvu6r1E+PfFmhzKFvNO4t2gVmQkLdu0Yd31dUw7NA1ngzPftf6OCM+I4m6WJEkl3MM4c2+iKErK755PArYrijJdCDGp6PlrD+F7HiujxcjHRz8mecVSXthmJbZeXVYEDab54RyEegsFaedw1Rey36kmY7Rr8fN9hexkP34Te8mwV2jXrh0VIirw9sG3WRe1jto+tZnRcAauBtfibpokSc+ARzEs0wloXPR4MbCLpyzcb+fcZvzu8TjsOc3YzVZiq9Vmi/9Aql/LA5ZSkJpCKYdMjmgrMNbuN9xsxpGQ4sZW7UGsBjX9evXF7Gqm18ZexGbFMqLKCEZUHiHXXpck6bH5t7dBKsAWIcRxIcTwojIvRVFuFz1OALzu9UYhxHAhxDEhxLHk5OR/WY2H58DNA/Tc2BP7o5cZuxFuhFdnT6mBBCdkYzQuoDAjhSpuCcQKT/r7ZuJseZNLRgO/6I6jd3Vg6LBhHDYdpu8vfckz5bGg5QJGRYySwS5J0mP1b8/c6yuKclMI4QlsFUJc+v2LiqIoQgjlXm9UFGUeMA8gMjLynsc8bhuiNjB5/2SapXjx/JosYkMqczR4KD4ZORgLFqEqzKOhbyzns7xpV642qqQKbNNHEatJIygoiJYdWzL11FR2xe2ikX8jptabiovBpbibJUnSM+hfhbuiKDeL/k4SQvwM1AQShRA+iqLcFkL4AEkPoZ6P3M9Xf+btA2/TsbA8/ZZcI8YvnNOhw3DJy0KVuxC10Ux9nxvEZXvQqPQobqbDXsMJTBorLVu0RBOsYcD2AaQXpMvZMJIkFbt/PCwjhLATQjj85zHQEjgHrAcGFh02EFj3byv5qK24tIK3DrxF/9Qy9JsfTYx7eS6Ufh4nUyq2ObMxGc24upq5nVeZyj6T2J+bxFbdGRzdHOg7tD97NXsZtnUYNhoblrRdQv/w/jLYJUkqVv/mzN0L+LkoxDTAUkVRNgshjgIrhRBDgVig57+v5qPz/fnv+eLQR0w57EOlPec5V64pCb5dcLHEocpZQppRT7aNE7Xsw1FrKvIzJ8hVF1KnenVuB6UxYO8Ack25dA3ryoTICdhp7Yq7SZIkSf883BVFuQ5UuUd5KtDs31TqcVlwdgGrtn7BzF9scEjOYnPjiegIxJ1L5OesJaXQhly9Gz1823BR5HJWfQpHILxleT5PmM3NEzep51ePcdXHUcalTHE3R5Ik6b+eyTtUFUXhm1NzuLBkFtO3aYn1b8Spei3RK1DBsIZTaXEUFtpgZ+dDS8+2bFddJVWdQ5Cjnm2h5/ju2mrKuJRhbvO51PWrW9zNkSRJusszF+6KojBr/yfoPvuOnokVOFSjD1aNM8G6Y/jpVrMxIQi9EYKcKuDqUYV1mlNoUNCWM/Np4Wo8Cz15r+57dAztKKc3SpL0xHqmwl1RFGZtnEzIZ/vJ8BjJmUrh2GqTqamfyqEMHZcTPNAKI1VcG3DL3ZYL6kt4Otiwxf8g1wuuMzB8ICMjRmKrtS3upkiSJP2pZybcrYqVrxePpcz3yUSXfg2LRk2E7WLycs/za5I3FqHGweBNLc/GHLC5ToEqF/uyBuYZl+Gp9mRhq4VyzXVJkp4az0S4W6wWZn30HCG7PIgqMxy9Op4Kylccv+WByepNvt6Vlo6NiHa2sE1zGSdbGy4EXeV4/nE6hnZkUs1JOOgcirsZkiRJD6zEh7vJYmL+uO743qhDTKmqOJjWkJd1haMWL7LsHKimCcfNrTy7tZfJV5lwCXNmqWUZWquWTxt9SsuglsXdBEmSpL+tRId7Rk42P7/QC3trJ246ZSGyZpFitVJg54SDjQvdbdpywnCLk5pz2NjrOON1livmK9Tzq8fUulPxsPUo7iZIkiT9IyU23C/ExXJm1EvkOdSkQNkG+fmobdXkeHjTurAueXb2bNCcxaSyEOsZy3Hb41RwqcCsiFk08Gsg7zCVJOmpVuLCvcBsZsXsWWh/O8htBz2Yj+FkoxDr60dzJRJH/DjicJnbmhtkGzI56HaYEL8Q5lSZQ13fujLUJUkqEUpEuCtWK+bkFA7t2U/C4iXk2DpTYJ+DWu2K2seAjWslemRX4pjmCjfURzGqjJx3OY9zqDOfVf2MWt61ZKhLklSiPNXhnrN7NwnTPiA1I4vLPv7YFToQ56xBUZKxdXElN8iLxtl1OZF/g9X6Q5iFmWiXaAIqBjA5fLKc2ihJUon1VIe7cHLiTGgYjudvkGMTSLLhMoraFuHvSxlNc2KyYvhZdxKrUMj3yadevXq8UeYNubiXJEkl3lMd7mt/XYPNDQ1n/N1RLJfJd7FFY18dkz6TPaqzADgEO9KlVRdCvUOLubaSJEmPz1Md7mkJ6Rjt47EoVhJKOWJrWw4N+Tg4uREREUG1iGq4uMidkCRJevY81eGeXdad7MIYMj2DCLT1o2LFilSuXJmAgAB5gVSSpGfaUx3uQ7uN4DenbURWrUqZMmXQaJ7q5kiSJD00T3Uaenl58Vy/fsVdDUmSpCfOP95DVZIkSXpyyXCXJEkqgWS4S5IklUAy3CVJkkogGe6SJEklkAx3SZKkEkiGuyRJUgkkw12SJKkEEoqiFHcdEEIkA7H/8O3uQMpDrM7TTvbHH8n++H+yL/6oJPRHKUVR7rkf6BMR7v+GEOKYoiiRxV2PJ4Xsjz+S/fH/ZF/8UUnvDzksI0mSVALJcJckSSqBSkK4zyvuCjxhZH/8keyP/yf74o9KdH889WPukiRJ0t1Kwpm7JEmS9D9kuEuSJJVAT3W4CyFaCyEuCyGuCSEmFXd9HjchxLdCiCQhxLnflbkKIbYKIa4W/f1MbCIrhAgQQuwUQlwQQpwXQowtKn9W+8MghDgihDhd1B/vFpUHCyEOF/3MrBBC6Iq7ro+LEEIthDgphNhY9LxE98VTG+5CCDUwC2gDhAN9hBDhxVurx24R0Pp/yiYB2xVFCQO2Fz1/FpiB8YqihAO1gVFF/x6e1f4oBJoqilIFiABaCyFqAzOAzxVFKQ2kA0OLr4qP3Vjg4u+el+i+eGrDHagJXFMU5bqiKEZgOdCpmOv0WCmKsgdI+5/iTsDioseLgc6Ps07FRVGU24qinCh6nM2dH2I/nt3+UBRFySl6qi36owBNgZ+Kyp+Z/hBC+APtgAVFzwUlvC+e5nD3A+J+9zy+qOxZ56Uoyu2ixwmAV3FWpjgIIYKAqsBhnuH+KBqGOAUkAVuBKCBDURRz0SHP0s/MF8BEwFr03I0S3hdPc7hLf0G5M8/1mZrrKoSwB1YDLyuKkvX71561/lAUxaIoSgTgz53/6ZYr3hoVDyFEeyBJUZTjxV2Xx0lT3BX4F24CAb977l9U9qxLFEL4KIpyWwjhw52ztmeCEELLnWD/UVGUNUXFz2x//IeiKBlCiJ1AHcBZCKEpOmN9Vn5m6gEdhRBtAQPgCMykhPfF03zmfhQIK7rirQN6A+uLuU5PgvXAwKLHA4F1xViXx6ZoDHUhcFFRlM9+99Kz2h8eQgjnosc2QAvuXIfYCXQvOuyZ6A9FUV5XFMVfUZQg7uTEDkVR+lHC++KpvkO16DfxF4Aa+FZRlGnFW6PHSwixDGjMnaVLE4G3gbXASiCQO8so91QU5X8vupY4Qoj6wF7gLP8/rvoGd8bdn8X+qMydi4Rq7pzErVQU5T0hRAh3Jh+4AieB/oqiFBZfTR8vIURjYIKiKO1Lel881eEuSZIk3dvTPCwjSZIk3YcMd0mSpBJIhrskSVIJJMNdkiSpBJLhLkmSVALJcJckSSqBZLhLkiSVQP8Hq0X0aSn0mAAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dfs = analysis.trial_dataframes\n",
    "\n",
    "# Plot by epoch\n",
    "ax = None  # This plots everything on the same plot\n",
    "for d in dfs.values():\n",
    "    ax = d.episode_reward_mean.plot(ax=ax, legend=False)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>config/lr</th>\n",
       "      <th>config/gamma</th>\n",
       "      <th>config/model</th>\n",
       "      <th>episode_reward_mean</th>\n",
       "      <th>timesteps_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.999</td>\n",
       "      <td>{'fcnet_hiddens': [128, 128]}</td>\n",
       "      <td>301.46</td>\n",
       "      <td>32000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.999</td>\n",
       "      <td>{'fcnet_hiddens': [64, 64]}</td>\n",
       "      <td>301.79</td>\n",
       "      <td>32000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.999</td>\n",
       "      <td>{'fcnet_hiddens': [128, 128]}</td>\n",
       "      <td>302.49</td>\n",
       "      <td>32000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.990</td>\n",
       "      <td>{'fcnet_hiddens': [128, 128]}</td>\n",
       "      <td>309.04</td>\n",
       "      <td>32000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.990</td>\n",
       "      <td>{'fcnet_hiddens': [128, 128]}</td>\n",
       "      <td>300.21</td>\n",
       "      <td>33000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.990</td>\n",
       "      <td>{'fcnet_hiddens': [64, 64]}</td>\n",
       "      <td>311.17</td>\n",
       "      <td>33000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.999</td>\n",
       "      <td>{'fcnet_hiddens': [16, 16]}</td>\n",
       "      <td>301.40</td>\n",
       "      <td>35000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.990</td>\n",
       "      <td>{'fcnet_hiddens': [32, 32]}</td>\n",
       "      <td>306.52</td>\n",
       "      <td>35000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.999</td>\n",
       "      <td>{'fcnet_hiddens': [32, 32]}</td>\n",
       "      <td>307.41</td>\n",
       "      <td>35000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.990</td>\n",
       "      <td>{'fcnet_hiddens': [32, 32]}</td>\n",
       "      <td>303.03</td>\n",
       "      <td>36000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.999</td>\n",
       "      <td>{'fcnet_hiddens': [64, 64]}</td>\n",
       "      <td>310.99</td>\n",
       "      <td>36000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.990</td>\n",
       "      <td>{'fcnet_hiddens': [16, 16]}</td>\n",
       "      <td>300.93</td>\n",
       "      <td>37000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.990</td>\n",
       "      <td>{'fcnet_hiddens': [64, 64]}</td>\n",
       "      <td>303.51</td>\n",
       "      <td>37000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.990</td>\n",
       "      <td>{'fcnet_hiddens': [16, 16]}</td>\n",
       "      <td>301.73</td>\n",
       "      <td>39000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.999</td>\n",
       "      <td>{'fcnet_hiddens': [32, 32]}</td>\n",
       "      <td>302.42</td>\n",
       "      <td>44000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.999</td>\n",
       "      <td>{'fcnet_hiddens': [16, 16]}</td>\n",
       "      <td>302.52</td>\n",
       "      <td>46000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    config/lr  config/gamma                   config/model  \\\n",
       "15     0.0001         0.999  {'fcnet_hiddens': [128, 128]}   \n",
       "9      0.0003         0.999    {'fcnet_hiddens': [64, 64]}   \n",
       "13     0.0003         0.999  {'fcnet_hiddens': [128, 128]}   \n",
       "12     0.0003         0.990  {'fcnet_hiddens': [128, 128]}   \n",
       "14     0.0001         0.990  {'fcnet_hiddens': [128, 128]}   \n",
       "8      0.0003         0.990    {'fcnet_hiddens': [64, 64]}   \n",
       "1      0.0003         0.999    {'fcnet_hiddens': [16, 16]}   \n",
       "4      0.0003         0.990    {'fcnet_hiddens': [32, 32]}   \n",
       "5      0.0003         0.999    {'fcnet_hiddens': [32, 32]}   \n",
       "6      0.0001         0.990    {'fcnet_hiddens': [32, 32]}   \n",
       "11     0.0001         0.999    {'fcnet_hiddens': [64, 64]}   \n",
       "0      0.0003         0.990    {'fcnet_hiddens': [16, 16]}   \n",
       "10     0.0001         0.990    {'fcnet_hiddens': [64, 64]}   \n",
       "2      0.0001         0.990    {'fcnet_hiddens': [16, 16]}   \n",
       "7      0.0001         0.999    {'fcnet_hiddens': [32, 32]}   \n",
       "3      0.0001         0.999    {'fcnet_hiddens': [16, 16]}   \n",
       "\n",
       "    episode_reward_mean  timesteps_total  \n",
       "15               301.46            32000  \n",
       "9                301.79            32000  \n",
       "13               302.49            32000  \n",
       "12               309.04            32000  \n",
       "14               300.21            33000  \n",
       "8                311.17            33000  \n",
       "1                301.40            35000  \n",
       "4                306.52            35000  \n",
       "5                307.41            35000  \n",
       "6                303.03            36000  \n",
       "11               310.99            36000  \n",
       "0                300.93            37000  \n",
       "10               303.51            37000  \n",
       "2                301.73            39000  \n",
       "7                302.42            44000  \n",
       "3                302.52            46000  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analysis.dataframe().sort_values(['timesteps_total','episode_reward_mean'])[['config/lr', \n",
    "                                                                             'config/gamma', \n",
    "                                                                             'config/model', \n",
    "                                                                             'episode_reward_mean', \n",
    "                                                                             'timesteps_total']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timesteps_total</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>config/lr</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0001</th>\n",
       "      <td>37875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.0003</th>\n",
       "      <td>33875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           timesteps_total\n",
       "config/lr                 \n",
       "0.0001               37875\n",
       "0.0003               33875"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analysis.dataframe()[['config/lr', 'timesteps_total']].groupby('config/lr').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timesteps_total</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>config/gamma</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.990</th>\n",
       "      <td>35250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.999</th>\n",
       "      <td>36500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              timesteps_total\n",
       "config/gamma                 \n",
       "0.990                   35250\n",
       "0.999                   36500"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analysis.dataframe()[['config/gamma', 'timesteps_total']].groupby('config/gamma').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timesteps_total</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>config/model</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>{'fcnet_hiddens': [128, 128]}</th>\n",
       "      <td>32250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>{'fcnet_hiddens': [64, 64]}</th>\n",
       "      <td>34500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>{'fcnet_hiddens': [32, 32]}</th>\n",
       "      <td>37500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>{'fcnet_hiddens': [16, 16]}</th>\n",
       "      <td>39250</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               timesteps_total\n",
       "config/model                                  \n",
       "{'fcnet_hiddens': [128, 128]}            32250\n",
       "{'fcnet_hiddens': [64, 64]}              34500\n",
       "{'fcnet_hiddens': [32, 32]}              37500\n",
       "{'fcnet_hiddens': [16, 16]}              39250"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aframe = analysis.dataframe()\n",
    "aframe['config/model'] = aframe['config/model'].astype(str)\n",
    "aframe[['config/model', 'timesteps_total']] \\\n",
    "    .groupby(['config/model']) \\\n",
    "    .mean() \\\n",
    "    .sort_values('timesteps_total') \\\n",
    "    .head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAJcCAYAAADtmzAcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2lklEQVR4nO3de7wXVb3/8deHDQJeEO8ieEstwzuiWFZeUkTTtKy0i1KanlIrPWVqN9Oy7NjJS6ZFSWI3NfuVZN7wrhkCmqKoKcdjCWqogIgml70/vz++A37lDIOcw3fvjfN69piH+7tmzcyarbu99nutWROZiSRJkl6vR1c3QJIkqTuykyRJklTCTpIkSVIJO0mSJEkl7CRJkiSVsJMkSZJUwk6SJBUiYm5EvKWr2yGpe7CTpNqKiCcjYp8lyj4ZEXd1VZtWVhGREbFlV7djaSLitoj49LLqZebqmflEZ7RJUvdnJ0nqYtGwQn8WI6JtRZ6vlSKip22Q1B3ZSZKWIiJOjojfLVF2QUScX3x9W0R8NyImRMSciLg6ItZuqrtbRNwdEbMj4oGI2LNp320RcVZE/Bl4BXjLGzjfbyPi2Yh4MSLuiIhtmvZdGhEXR8S1EfEysFdEvC8i/lqc66mI+GZT/c2K9OdTxb5ZEfGZiNglIiYXbb5wiXs/KiIeKereEBGbFuV3FFUeKIarDivKD4yI+4tz3R0R2zed68mIOCUiJgMvl3VSivYdFxGPR8RLEfGtiNiiONeciLgyIlYp6q4VEddExHNF+66JiEHFvrOAdwMXFu27sOn8x0fE48DjTWVbRsQqRds/V5S3RcSfI+Iblf/RSHpzyUw3t1puwJPAPkuUfRK4q/h6APAy0L/43BOYAexcfL4NmA5sC6wG/A74ZbFvIPACcACNP0b2LT6v13TsP4BtivP2qjpfccxRwBpAb+A84P6mfZcCLwK7F9frA+wJbFd83h74J3BIUX8zIIEfF3WHA68CfwDWL9o/A9ijqH8wMBV4e9HerwF3N10/gS2bPu9UHD8MaANGFt/v3k3f+/uBjYG+S/n3k8DVQL/i+zQPuBl4C7Am8DAwsqi7DnAosGrxPfot8Iemc90GfLrk/OOAtRe1ofk+in8Ps4p7/iowHmjr6v9u3dzcOm/r8ga4uXXVVvyingvMbtpeoegkFXWuA44pvj4QeLhp323A2U2fBwPzi07BKcAvlrjeDU2/1G8Dzlxi/1LPV9L2/sUv9DWLz5cCly3jfs8Dzi2+3qw4fmDT/heAw5o+/w44sen7cHTTvh7F92rT4vOSnaSLgW8tcf2/8Vqn60ngqGW0N4Hdmz7fC5zS9Pk/gfOWcuyOwKwlvrdlnaS9S8qa7+OLRbtnAVt19X+zbm5unbs53Ka6OyQz+y/agOOW2D8G+ETx9SeAXyyx/6mmr/9OIxFaF9gU+HAx1DQ7ImYD76KRTpUdW3m+Yrjn7Ij4r4iYQ6OTQXGt0vNFxLCIuLUYgnoR+MwS9aGRLi3yr5LPqxdfbwqc33QvM4GgkTiV2RT44hL3vzGw0dLauxRvqH0RsWpE/CQi/l58f+4A+sey52Ytqw1jaNzLtZn5+Btor6Q3ETtJUrU/ANtHxLY0kqRfLbF/46avNwEWAM/T+OX7i+YOWGaulplnN9XPkust7XwfozHktQ+NoabNijpRcb5fA2OBjTNzTRpDa8H/zlPAvy1xP30z8+6K+mctUX/VzPxNRXv/L74IvA0Ylpn9gPcU5Yvud2nXWlYbLgKuAfaLiHf9n1spaaViJ0mqkJmvAlfR6HBMyMx/LFHlExExOCJWBc4ErsrMduCXwEERsV+RAvWJiD0XTSausLTzrUFjTs4LNObdfOcNNH8NYGZmvhoRu9LoaP1v/Rg4bdFk8YhYMyI+3LT/nzTmCi3yU+AzRZoVEbFaMZF8jf9DG6qsQSNZml1Mdj99if1Ltm+ZIuIIYGca89Q+D4yJiNUrD5L0pmInSVq2MTQmQC851EZRdinwLI0J0J8HyMynaCQ/XwGeo5GsnMyyf+ZKzwdcRmP4bTqNCcvj30C7jwPOjIiXgG8AV76BY0pl5u+B7wGXF8NZDwH7N1X5Jo1OxOyI+EhmTgKOAS6kMZ9nKo3ORqucB/SlkbqNB65fYv/5wIeKJ98uWNbJImKT4pxHZubczPw1MAk4d0U2WlL3FpkrMvGW3nyKX5iPAhtm5pym8ttoPH32sxV0nRV6PknS/41JklQhGos8/jtweXMHSZL05ucqs9JSRMRqNOay/B0Y0cXNkSR1MofbJEmSSjjcJkmSVKLbDreN2+AwIy6pC+w55btd3QSptnqt+5b/7Vpm/ysLnn+i037Xdva9rQgmSZIkSSXsJEmSJJXotsNtkiSpxTrau7oF3ZpJkiRJUgmTJEmS6io7uroF3ZpJkiRJUgmTJEmS6qrDJKmKSZIkSVIJkyRJkmoqnZNUySRJkiSphEmSJEl15ZykSiZJkiRJJUySJEmqK+ckVTJJkiRJKmEnSZIkqYTDbZIk1ZUvuK1kkiRJklTCJEmSpLpy4nYlkyRJkqQSJkmSJNWVi0lWMkmSJEkqYZIkSVJN+YLbaiZJkiRJJUySJEmqK+ckVTJJkiRJKmGSJElSXTknqZJJkiRJUgmTJEmS6sp3t1UySZIkSSphkiRJUl05J6mSSZIkSVIJO0mSJEklHG6TJKmuXEyykkmSJElSCZMkSZLqyonblUySJEmSSpgkSZJUV85JqmSSJEmSVMIkSZKkmsr0tSRVTJIkSZJKmCRJklRXPt1WySRJkiR1qYjoExETIuKBiJgSEWcU5XtHxH0R8VBEjImInkV5RMQFETE1IiZHxJCmc42MiMeLbWRT+c4R8WBxzAUREctql50kSZLqqqOj87Zq84C9M3MHYEdgRES8ExgDHJ6Z2wJ/BxZ1evYHtiq2Y4GLASJibeB0YBiwK3B6RKxVHHMxcEzTcSOW1Sg7SZIkqUtlw9ziY69iawfmZ+ZjRfk44NDi64OBy4rjxgP9I2IAsB8wLjNnZuas4pgRxb5+mTk+MxO4DDhkWe2ykyRJUl1lR6dtEXFsRExq2o5tbkpEtEXE/cAMGp2bCUDPiBhaVPkQsHHx9UDgqabDpxVlVeXTSsorOXFbkiS1XGaOAkZV7G8HdoyI/sDvgW2Aw4FzI6I3cCONdKnT2EmSJKmuOrrfOkmZOTsibgVGZOb3gXcDRMRw4K1Ftem8lioBDCrKpgN7LlF+W1E+qKR+JYfbJElSl4qI9YoEiYjoC+wLPBoR6xdlvYFTgB8Xh4wFjiyectsNeDEznwFuAIZHxFrFhO3hwA3FvjkRsVvxVNuRwNXLapdJkiRJ6moDgDER0UYjwLkyM6+JiHMi4sCi7OLMvKWofy1wADAVeAX4FEBmzoyIbwETi3pnZubM4uvjgEuBvsB1xVbJTpIkSXXVTRaTzMzJwE4l5ScDJ5eUJ3D8Us41GhhdUj4J2HZ52uVwmyRJUgmTJEmS6mrZizzWmkmSJElSCZMkSZLqqpvMSequTJIkSZJKmCRJklRXzkmqZJIkSZJUwiRJkqS6MkmqZJIkSZJUwiRJkqSayux+L7jtTkySJEmSSpgkSZJUV85JqmSSJEmSVMIkSZKkunLF7UomSZIkSSXsJEmSJJVwuE2SpLpy4nYlkyRJkqQSJkmSJNWVE7crmSRJkiSVMEmSJKmunJNUySRJkiSphEmSJEl15ZykSiZJkiRJJUySJEmqK+ckVTJJkiRJKmGSJElSXZkkVTJJkiRJKmGSJElSXfl0WyWTJEmSpBImSZIk1ZVzkiqZJEmSJJWwkyRJklTC4TZJkurKiduVTJIkSZJKmCRJklRXTtyuZJIkSZJUwiRJkqS6ck5SJZMkSZKkEiZJkiTVlXOSKpkkSZIklTBJkiSprkySKpkkSZIklTBJkiSprjK7ugXdmkmSJElSCZMkSZLqyjlJlUySJEmSSpgkSZJUVyZJlUySJEmSSpgkSZJUV767rZJJkiRJUgk7SZIkSSUcbpMkqa6cuF3JJEmSJKmESZIkSXXla0kqmSRJkiSVMEmSJKmunJNUySRJkiSphEmSJEl1ZZJUySRJkiSphEmSJEl15WtJKpkkSZIklTBJkiSpprLDdZKqmCRJkiSVMEmSJKmufLqtkkmSJElSCZMkSZLqyqfbKpkkSZIklbCTJEmSVMLhNkmS6solACqZJEmSJJUwSZIkqa5cAqCSSZIkSVIJkyRJkurKJKmSSZIkSVIJkyRJkuoqfbqtikmSJElSCZMkSZLqyjlJlUySJEmSSpgkSZJUV664XckkSZIkqYRJkpaqR+9eDL36m/RYpRfR1oN/XnMPT5zzWwC2OO0wNjhoN7I9mTbmRp762fVseOi72OyE90ME7XP/xSNfvoS5D/8dgHdN/CELX34V2jvIhe3cs99XAOjZfzW2H3UifTdej3899RyTjzmPhS++3GX3LHUH8+bNZ+TxJzN/wQLaF7az717v4oRPH8E9997P9y/8GQsWLGTw27bkzNNOomfPNibcN5nPn3oGAwdsCMA+e7yTzx71cQDuGj+Js8/7Me0dHRx60Ag+fcRHAJj29LOcfPrZzH5xDoPfthVnf+NL9OrVq8vuWV0knZNUxU6Slqpj3gLu/eCZtL8yj+jZxi5/PIMXbrmf1bYaSJ+N1uXu3f8dMum1bj8A/vX3GUw65AwWvvgy6+y9I4P/8xgm7P+1xee794NnsmDmS6+7xuafO4SZdz7Ekz+8ms0+dzCbfe5gpn771516n1J3s8oqvRh9wdmsumpfFixcyJGf/RK7D9uZr3z7P7nk/O+y2SaDuPCnl3H1dTdx6EH7ATBkh2256JwzXnee9vZ2vv2fP+Kn532HDddfl8M+/QX2etcwtth8U869eDRHHHYIB+yzJ2f8xw/53TU3cPgHDuyK25WIiD7AHUBvGn2TqzLz9Ih4L3AOjZGvucAnM3NqRPQGLgN2Bl4ADsvMJ4tznQYcDbQDn8/MG4ryEcD5QBvws8w8e1ntcrhNldpfmQdA9GojevYkMxn0yX154j+vWry+xoLn5wDw4qTHFqdAL977OL0HrLPM8683YihPX3E7AE9fcTvr779LK25DWqlEBKuu2heAhQsXsnDhQtp69KBXz55stskgAN6xyxBuuu2uyvM8+MhjbDJoIzYeOIBevXqx/3v34JY7x5OZ3HPvAwzf890AHHzAPtxyx19ae1Pqnjqy87Zq84C9M3MHYEdgRETsBlwMfDwzdwR+DSz6y/toYFZmbgmcC3wPICIGA4cD2wAjgIsioi0i2oAfAfsDg4GPFnUr2UlStR7Bbjd/jz2m/JQXbp/MnPum0nfTDdjwkHcy7IbvsNOvT2XVzTf8H4cN/NhevHDL/a8rG3LFVxl243cZeMR7F5etst6azJ8xG4D5M2azynprtvJupJVGe3s7h448nvcc+FHesctObDf4bbS3d/DQI48BcONtd/HsjOcX13/goUf44Mjj+MwXv87UJxrD3DOee54N119vcZ0N1l+XGc+9wOwX57DG6qvRs2dbo3y9RrnUVbJhbvGxV7FlsfUrytcEni6+PhgYU3x9FfDeiIii/PLMnJeZ/w1MBXYttqmZ+URmzgcuL+pW6vThtogYlZnHLmXfscCxAF9YY2fe13eLTm2bSnQk4997Cj37rcoOl36J1bbemB69e9Hx6gLu2e8rrH/Argw+7zNMOvibiw9Za/dt2OhjezPp/d9YXDbxoG8w79lZ9Fq3Hztf+TVefvxpZo9/5H9ez9VfJQDa2tr43ZgfMeeluXzhtG8x9b//zjlnnsp/XDCK+QsW8M5dh9CjR+Pv3MFv24JxvxvDqqv25Y67J/D5087k2isu6eI7kF6v+Xd8YVRmjmra3wbcC2wJ/Cgz74mITwPXRsS/gDnAbkX1gcBTAJm5MCJeBNYpysc3XWNaUcai+k3lw5bV5pYkSRGx9lK2dYADlnZcZo7KzKGZOdQOUveycM4rzLprCuvutQPznn6Bf157DwAzrp3A6oM3XVxv9cGbMPgHx/LAyHNYMGvu4vJ5z84CGkNzM66dwJo7Nf79zn/uRVZZvz8Aq6zfn/nF0J2khn5rrM6uQ7bnrvGT2HHbt3PZxd/n8p+dz847bMtmmzT+v3/11VZbPDz3nnfuysKFC5k1+0XWX29dnp3x3OJz/XPG86y/3jr0X7MfL819mYUL2xvlzzXKVT/Z0dF5W9Pv+GIb9bq2ZLYXw2qDgF0jYlvgJOCAzBwE/Bz4QWd+f1o13PYcMIlGj3DRNqnY1m/RNbWC9VpnDXr2WxWAHn16sfYe2/Hy1KeZcf1E1t59GwDWeudgXvmvZwDoM3Addhj9RR46/ke88sQzi8/TY9XetK3WZ/HX6+y5PXMfbXTon7thEhsdtgcAGx22B89dP6nT7k/qrmbOms2clxp/ZLw6bx5/mfhXNt90Y16YNRuA+fPnM/pXv+UjhzT+5nz+hZlkkcI++PDf6Mik/5r92Hbrt/KPaU8z7elnWbBgAdfdfDt7vWs3IoJdh2zPjbfdCcDV197E3u9+R+ffqFQiM2cDt9KYP7RDZt5T7LoCeGfx9XRgY4CI6EljKO6F5vLCoKJsaeWVWjXc9gTw3sz8x5I7IuKpkvrqhnpvsBbbXHAc0daD6NGDf179F54fdx+z73mUbS/6HJv82/tof/lVHv73nwDwli9+iF5rrc7bv3c0wOJH/XuvtyY7/PxLAERbD579/Z954dYHAHjyh1ez3U9PZODH9uJf055n8jHnds3NSt3Icy/M4qvf/j7tHR1kR7Lf3u9mz92H8f0Lf8btd08gOzo47APvY9jOOwJw4613ccXv/0Rbzzb6rLIK55xxKhFBz55tfOWkz/Jv//412tvb+cCBw9nyLY3k96TPHsXJp5/ND0ddxtvfugUfPHB4F96xukw3WUwyItYDFmTm7IjoC+xLYzL2mhHx1sx8rChbNE9jLDAS+AvwIeCWzMyIGAv8OiJ+AGwEbAVMAALYKiI2p9E5Ohz42DLblS2YAxIRxwN3ZeYDJfs+l5k/XNY5xm1wWPf4NyfVzJ5TvtvVTZBqq9e6b4nOvN7LZx3Zab9rV/vqZUu9t4jYnsZE7DYao1xXZuaZEfEB4EygA5gFHJWZTxRLBvwC2AmYCRyemU8U5/oqcBSwEDgxM68ryg8AziuuMTozz1pWm1vSSVoR7CRJXcNOktR1Or2T9O1PdF4n6Wu/7NR7WxFa9nRbRGxN4/G6RbPKpwNjM7PkkSZJkqTupVVPt51CYw2CoDEWuGg88DcRcWorrilJkpZT91lMsltqVZJ0NLBNZi5oLiwmUk0BlrkUuCRJUldqVSepg8as8r8vUT6g2CdJkrpah7+Sq7Sqk3QicHNEPM5rK1xuQmMVzRNadE1JkqQVpiWdpMy8PiLeSuNdKc0TtydmZnsrrilJkpbTSjpXqLO08t1t2bQt+myuJ0mSVgot6SRFxHDgIuBxXlv2exCwZUQcl5k3tuK6kiRpOaTZRZVWJUnnA/tk5pPNhcVy4NcCb2/RdSVJklaIVnWSegLTSsqnA71adE1JkrQ8nJNUqVWdpNHAxIi4nNeebtuYxgvlLmnRNSVJklaYVj3d9t2IuBp4P/COong68PHMfLgV15QkSVqRWvZ0W9EZejgi1i4+z2zVtSRJ0vJLF5Os1Kp3t20SEZdHxAzgHmBCRMwoyjZrxTUlSZJWpFYlSVcA59EYXmsHiIg24MM0Xny7W4uuK0mS3ignbldqSZIErJuZVzSvrp2Z7Zl5ObBOi64pSZK0wrQqSbo3Ii4CxvD6p9tGAn9t0TUlSdLyMEmq1KpO0pHA0cAZvPbutmnAH3EJAEmStBJo1RIA84GLi02SJHVHvpakUqvmJC1VRBzY2deUJElaXi1bJ6nCLsA1XXBdSZLUzDlJlVrWSYqIrYGDeW1O0nRgbGae3qprSpIkrSitWkzyFBrrIQUwodgC+E1EnNqKa0qSpOWTHdlp28qoVUnS0cA2mbmguTAifgBMAc5u0XUlSZJWiFZ1kjqAjYC/L1E+oNgnSZK62kqa8HSWVnWSTgRujojHeW0xyU2ALYETWnRNSZKkFaZV6yRdHxFvBXbl9RO3Jza/qkSSJHWhDgd3qrTs6bbM7ADGt+r8kiRJrdTpi0lKkiStDLpiMUlJktQdOHG7kkmSJElSCZMkSZLqyiSpkkmSJElSCZMkSZJqKtMkqYpJkiRJUgmTJEmS6so5SZVMkiRJkkqYJEmSVFcmSZVMkiRJkkqYJEmSVFNpklTJJEmSJKmESZIkSXVlklTJJEmSJKmESZIkSXXV0dUN6N5MkiRJkkrYSZIkSSrhcJskSTXlEgDVTJIkSZJKmCRJklRXJkmVTJIkSZJKmCRJklRXLgFQySRJkiSphEmSJEk15dNt1UySJEmSSpgkSZJUV85JqmSSJEmSVMIkSZKkmnJOUjWTJEmSpBImSZIk1ZVzkiqZJEmSJJUwSZIkqabSJKmSSZIkSVIJO0mSJEklHG6TJKmuHG6rZJIkSZJUwiRJkqSacuJ2NZMkSZKkEiZJkiTVlUlSJZMkSZKkEiZJkiTVlHOSqpkkSZIklTBJkiSppkySqpkkSZIklTBJkiSppkySqpkkSZIklTBJkiSprjK6ugXdmkmSJElSCZMkSZJqyjlJ1UySJEmSSthJkiRJKuFwmyRJNZUdTtyuYpIkSZJUwk6SJEk1lR2dt1WJiD4RMSEiHoiIKRFxRlF+Z0TcX2xPR8QfivKIiAsiYmpETI6IIU3nGhkRjxfbyKbynSPiweKYCyJimTGaw22SJKmrzQP2zsy5EdELuCsirsvMdy+qEBG/A64uPu4PbFVsw4CLgWERsTZwOjAUSODeiBibmbOKOscA9wDXAiOA66oaZZIkSVJNZUanbdXtyMzMucXHXsWWi/ZHRD9gb+APRdHBwGXFceOB/hExANgPGJeZM4uO0ThgRLGvX2aOz8wELgMOWdb3x06SJElquYg4NiImNW3HLrG/LSLuB2bQ6Ojc07T7EODmzJxTfB4IPNW0f1pRVlU+raS8ksNtkiTVVGcuJpmZo4BRFfvbgR0joj/w+4jYNjMfKnZ/FPhZ61v5eiZJkiSp28jM2cCtNOYMERHrArsCf2qqNh3YuOnzoKKsqnxQSXklO0mSJNVUdkSnbVUiYr0iQSIi+gL7Ao8Wuz8EXJOZrzYdMhY4snjKbTfgxcx8BrgBGB4Ra0XEWsBw4IZi35yI2K14qu1IXpsEvlQOt0mSpK42ABgTEW00ApwrM/OaYt/hwNlL1L8WOACYCrwCfAogM2dGxLeAiUW9MzNzZvH1ccClQF8aT7VVPtkGdpIkSaqtzGXX6QyZORnYaSn79iwpS+D4pdQfDYwuKZ8EbLs87XK4TZIkqYRJkiRJNeW726qZJEmSJJUwSZIkqaZMkqqZJEmSJJWwkyRJklTC4TZJkmqquywB0F2ZJEmSJJUwSZIkqaacuF3NJEmSJKmESZIkSTWVaZJUxSRJkiSphEmSJEk1lR1d3YLuzSRJkiSphEmSJEk11eGcpEomSZIkSSVMkiRJqimfbqtmkiRJklTCJEmSpJpyxe1qJkmSJEklTJIkSaqpzK5uQfdmkiRJklTCTpIkSVIJh9skSaopJ25XM0mSJEkqYZIkSVJN+VqSaiZJkiRJJUySJEmqKV9LUs0kSZIkqYRJkiRJNeViktVMkiRJkkqYJEmSVFM+3VbNJEmSJKmESZIkSTXl023VTJIkSZJKmCRJklRTPt1WzSRJkiSphEmSJEk15dNt1ZaZJEVEW0Q82hmNkSRJ6i6WmSRlZntE/C0iNsnMf3RGowC+HNM661KSmjy40bu7uglSbS2cP71Tr+fTbdXe6HDbWsCUiJgAvLyoMDPf35JWSZIkdbE32kn6ektbIUmS1M28oU5SZt7e6oZIkqTO5cTtapWdpIh4CShbRSGAzMx+LWmVJElSF6vsJGXmGp3VEEmS1LlcS7Kai0lKkiSVcDFJSZJqyjlJ1UySJEmSSpgkSZJUUy4mWc0kSZIkqYRJkiRJNdXR1Q3o5kySJEmSSpgkSZJUU4lzkqqYJEmSJJUwSZIkqaY6XHK7kkmSJElSCZMkSZJqqsM5SZVMkiRJkkrYSZIkSSrhcJskSTXlEgDVTJIkSZJKmCRJklRTvpakmkmSJElSCZMkSZJqyjlJ1UySJEmSSpgkSZJUU85JqmaSJEmSVMIkSZKkmjJJqmaSJEmSVMIkSZKkmvLptmomSZIkSSVMkiRJqqkOg6RKJkmSJEklTJIkSaqpDuckVTJJkiRJKmEnSZIkqYTDbZIk1VR2dQO6OZMkSZKkEiZJkiTVlK8lqWaSJEmSVMIkSZKkmuoIlwCoYpIkSZK6VET0iYgJEfFAREyJiDOK8oiIsyLisYh4JCI+31R+QURMjYjJETGk6VwjI+LxYhvZVL5zRDxYHHNBxLJ7iCZJkiTVVDd6um0esHdmzo2IXsBdEXEd8HZgY2DrzOyIiPWL+vsDWxXbMOBiYFhErA2cDgylcXv3RsTYzJxV1DkGuAe4FhgBXFfVKJMkSZLUpbJhbvGxV7El8FngzMzsKOrNKOocDFxWHDce6B8RA4D9gHGZObPoGI0DRhT7+mXm+MxM4DLgkGW1y06SJEk11dGJW0QcGxGTmrZjm9sSEW0RcT8wg0ZH5x5gC+Cwov51EbFVUX0g8FTT4dOKsqryaSXllRxukyRJLZeZo4BRFfvbgR0joj/w+4jYFugNvJqZQyPig8Bo4N2d0V4wSZIkqbY6ovO2NyozZwO30pgzNA34f8Wu3wPbF19PpzFXaZFBRVlV+aCS8kp2kiRJUpeKiPWKBImI6AvsCzwK/AHYq6i2B/BY8fVY4MjiKbfdgBcz8xngBmB4RKwVEWsBw4Ebin1zImK34qm2I4Grl9Uuh9skSaqpDrrNOkkDgDER0UYjwLkyM6+JiLuAX0XEScBc4NNF/WuBA4CpwCvApwAyc2ZEfAuYWNQ7MzNnFl8fB1wK9KXxVFvlk21gJ0mSJHWxzJwM7FRSPht4X0l5Ascv5VyjacxdWrJ8ErDt8rTLTpIkSTXVjdZJ6packyRJklTCTpIkSVIJh9skSaqp5Xk0v45MkiRJkkqYJEmSVFMdXd2Abs4kSZIkqYRJkiRJNeUSANVMkiRJkkqYJEmSVFM+3VbNJEmSJKmESZIkSTXl023VTJIkSZJKmCRJklRTJknVTJIkSZJKmCRJklRT6dNtlUySJEmSSpgkSZJUU85JqmaSJEmSVMJOkiRJUgmH2yRJqimH26qZJEmSJJUwSZIkqaayqxvQzZkkSZIklTBJkiSppjpcTLKSSZIkSVIJkyRJkmrKp9uqmSRJkiSVMEmSJKmmTJKqmSRJkiSVMEmSJKmmXCepmkmSJElSCZMkSZJqynWSqpkkSZIklTBJkiSppny6rZpJkiRJUgk7SZIkSSUcbpMkqaZcAqCaSZIkSVIJkyRJkmqqwyypkkmSJElSCZMkSZJqyiUAqpkkSZIklTBJkiSpppyRVM0kSZIkqYRJkiRJNeWcpGomSZIkSSVMkiRJqqmO6OoWdG8mSZIkSSVMkiRJqilX3K5mkiRJklTCJEmSpJoyR6pmkiRJklTCTpIkSVIJh9skSaopF5OsZpIkSZJUwiRJkqSacgmAaiZJkiRJJUySJEmqKXOkaiZJkiRJJUySJEmqKZ9uq2aSJEmSVMIkSZKkmvLptmomSZIkSSVMkiRJqilzpGomSZIkSSVMkiRJqimfbqtmkiRJklTCJEmSpJpKZyVVMkmSJEkqYSdJkiSphMNtkiTVlBO3q5kkSZIklTBJkiSppnwtSTWTJEmSpBImSZIk1ZQ5UjWTJEmSpBImSZIk1ZRzkqqZJEmSJJUwSZIkqaZcJ6maSZIkSVIJO0laqlV6r8IvrvspV9x8KVfd/ks+c/LRr9v/5W+fyJ//a9ziz5/4t8P43R2/5IpbxvDj357PgEEbLN43afodXH7TpVx+06WcN+Z7i8s32mQAl107iqv/cgVn/+RMevYy3JR69+7NX/58DfdOGscD99/C6d/4IgCjfvJ97p00jvvuHccVl49itdVWBeDIIz7CM9MnM2nijUyaeCNHfeqji891xBEf5pEpd/HIlLs44ogPLy4fstN2/PW+m3j04bs49wdndu4NqtvITvzfyshOkpZq/rz5HHvo5znsvZ/k8PeO5J17DWO7IdsAMHiHrVljzTVeV//Rhx7n4/sdzWF7j+Tma27lC18/fvG+ea/O4/B9Psnh+3ySE0eesrj8C1/7LL/6yRUc/I7DeGn2S3zgYwd2zs1J3di8efPYZ/hH2Hnovuw8dDj7Dd+TYbsO4Ytf+iY7D92XITvvy1P/mM7xx31q8TFX/nYsQ3cZztBdhjP6578BYK21+vP1r57EO991IO/Y/X18/asn0b//mgD86MLv8pnPfJmtB7+LrbbcnBH77dUl9yoBRESfiJgQEQ9ExJSIOKMovzQi/jsi7i+2HYvyiIgLImJqREyOiCFN5xoZEY8X28im8p0j4sHimAsiIpbVLjtJqvSvV/4FQM9ePenZsyeZSY8ePTjxG8dz/rcuel3dSX++j1f/NQ+AyfdOYYMB6y3z/LvsvjM3XXMbAH+88lr2HPGeFXsD0krq5ZdfAaBXr5707NWLzOSll+Yu3t+nbx8yq/86Hz58D266+U5mzZrN7NkvctPNd7Lffnuy4Ybrs0a/Nbhnwn0A/OJXV/H+949o3c2o2+roxG0Z5gF7Z+YOwI7AiIjYrdh3cmbuWGz3F2X7A1sV27HAxQARsTZwOjAM2BU4PSLWKo65GDim6bhl/kffkk5SRKwaEV+OiJOL3uEnI2JsRPxHRKzeimuqNXr06MHlN13KzQ9dw/g7JvLQXx/msKMO5fYb7uL5GS8s9bhDPnYQf75l/OLPq/RehV/dcAlj/jSKPUe8G4D+a6/JS3Pm0t7eDsA/n3mO9d9Ax0qqgx49ejBp4o08M30yN998BxMm/hWAn/30B0x/6n62ftuWXPij0Yvrf/ADBywehhs0aCMABm60IdOmPb24zvTpzzBwow0ZuNGGTJ/2zGvl0xrlUlfJhkV/BfQqtqq/Ag4GLiuOGw/0j4gBwH7AuMycmZmzgHE0OlwDgH6ZOT4bf11cBhyyrHa1Kkm6FNgA2Bz4EzAUOAcIit5emYg4NiImRcSk5195tkVN0/Lo6Ojg8H0+yX47fYBtdxrMkN12YN+D9uLyS65a6jEHHDqcwTtszZiLfv1a2dBD+fh+R/OVz36Tk7/1BQZtOrAzmi+ttDo6Ohi6y3A23XwouwzdiW22eRsAnz7m39l40yE88ujjfOTD7wfgmj+NY4utdmPIzvty00138PNLzuvClmtl0plzkpp/xxfbsc1tiYi2iLgfmEGjo3NPseusYkjt3IjoXZQNBJ5qOnxaUVZVPq2kvFKrOklvzcwvAscD2wCfy8w7gS8DOyztoMwclZlDM3Pouqv6V013MnfOXCb9+T6G7j6EjTcfxNjxV/CniVfRp28frv7LFYvrDXv3UI7+wkhOHPllFsxfsLj8uWefB2D6P55m0t1/ZevttmL2zBdZo9/qtLW1AbDBgPWY8cxznXtjUjf34otzuO32P7Pf8D0Xl3V0dHDllVfzwQ+8D4CZM2cxf/58AC4Z/WuGDNkOgOlPP7s4VQIYOHAA059+lulPP8vAQQNeKx/UKJdaqfl3fLGNWmJ/e2buCAwCdo2IbYHTgK2BXYC1gVOWPG8rtXROUhFpXVv8c9HnlXOKew2ttU5/Vu/XGB3t3WcVhr1nFx6Z/Df23f79vG+XD/G+XT7Eq/96lYPfcRgAb9t2K756zpc5aeQpzHp+9uLzrLHmGvRapRfQGGLbcZfteOKxJwGYdPd97HPgngAc9JEDuO2GOzvt/qTuat1112bNNfsB0KdPH/Z573t47LEn2GKLzRbXOejA4fztb1MB2HDD9V8rP2g4jz7aKL/xxtvZd5/30L//mvTvvyb77vMebrzxdp59dgYvzXmJYbs25roe8fEP8cc/3tBJdydVy8zZwK3AiMx8phhSmwf8nMY8I4DpwMZNhw0qyqrKB5WUV2rV89aTImL1zJybmUctKoyILYCXWnRNrWDrrr8OZ17wNXq09aBHjx6MG3sLd467e6n1T/rG8ay6Wl/+46ffBuDZ6f/kxJGn8JatNuWr53yZ7OggevTg5z/85eJO0vnfupizf3IGx516LH976DH+8OtrOuPWpG5twIANGH3JebQVP3tXXfVH/nTtTdx+6+9Zo9/qRASTJz/M8SecBsDnTjiKAw8czsKF7cyaOZujPn0iALNmzeas75zH+Lv/BMC3zzqXWbNmA3DC577CJZecS98+fbj+hlu57vpbuuJW1cW6y2KSEbEesCAzZ0dEX2Bf4HsRMSAznymeRDsEeKg4ZCxwQkRcTmOS9otFvRuA7zRN1h4OnJaZMyNiTjEZ/B7gSOCHy2zXsp6OWNEiIvINXHSnDXc3cZK6wIMzn+zqJki1tXD+9GU+lr4ijdzs0E77XTvmyd8t9d4iYntgDNBGY5Trysw8MyJuAdajMaf5fuAzmTm36DRdSOMJtVeAT2XmpOJcRwFfKU59Vmb+vCgfSmPOdF/gOhpTgSrvv2Ur90XE1jRmny+aGDUdGJuZj7TqmpIk6Y3r6OSgZGkyczKwU0n53kupnzTmPZftGw2MLimfBGy7PO1q1RIApwCX0+j5TSi2AH4TEae24pqSJEkrUquSpKOBbTJzQXNhRPwAmAKc3aLrSpKkN6h75EjdV6uebusANiopH0D3mScmSZK0VK1Kkk4Ebo6Ix3ltUadNgC2BE1p0TUmStBw6zJIqtaSTlJnXR8Rbaaxn0Dxxe2JmtrfimpIkSStSy55uozHU2bx4ZOJQmyRJ3UaaJFVqSScpIoYDFwGP89qKloOALSPiuMy8sRXXlSRJWlFalSSdD+yTmU82F0bE5sC1wNtbdF1JkvQGObxTrVVPt/Xk9W/bXWQ60KtF15QkSVphWpUkjQYmFu9UWfR028bA4cAlLbqmJElaDj7dVq1VT7d9NyKuBt4PvKMong58PDMfbsU1JUmSVqSWPd1WdIYejoi1i88zW3UtSZK0/Hy6rVqr3t22SURcHhEzgHuACRExoyjbrBXXlCRJWpFaNXH7CuD3wIDM3Cozt6TxSpI/0HjxrSRJUrfWqk7Supl5RfPq2pnZnpmXA+u06JqSJGk5dHTitjJq1ZykeyPiImAMr3+6bSTw1xZdU5IkaYVpVSfpSOBo4Axee3fbNOCPuASAJEndQqYTt6u0agmA+cDFxSZJkrTSadWcpKWKiAM7+5qSJOl/6iA7bVsZdXonCdilC64pSZK0XFq2mGREbA0czGtzkqYDYzPz9FZdU5IkvXEr61NnnaVVi0meQmM9pAAmFFsAv4mIU1txTUmSpBWpVUnS0cA2mbmguTAifgBMAc5u0XUlSdIb5GtJqrVqTlIHsFFJ+QBM9yRJ0kqgVUnSicDNEfE4ry0muQmwJXBCi64pSZKWw8r61FlnadU6SddHxFuBXXn9xO2Jza8qkSRJ6q5a9nRbZnYA41t1fkmS9H/jitvVumKdJEmSpG6vZUmSJEnq3nySqppJkiRJUgmTJEmSasp1kqqZJEmSJJWwkyRJklTC4TZJkmrKxSSrmSRJkiSVMEmSJKmmXEyymkmSJElSCZMkSZJqyjlJ1UySJEmSSpgkSZJUUy4mWc0kSZIkqYRJkiRJNdXh022VTJIkSZJKmCRJklRT5kjVTJIkSZJKmCRJklRTrpNUzSRJkiSphEmSJEk1ZZJUzSRJkiSphJ0kSZKkEg63SZJUU+likpVMkiRJkkqYJEmSVFNO3K5mkiRJklTCJEmSpJpKk6RKJkmSJEklTJIkSaopn26rZpIkSZJUwiRJkqSa8um2aiZJkiRJJUySJEmqKeckVTNJkiRJKmGSJElSTTknqZpJkiRJUgmTJEmSasoVt6uZJEmSJJWwkyRJklTC4TZJkmqqwyUAKpkkSZIklTBJkiSpppy4Xc0kSZIkqYRJkiRJNeWcpGomSZIkSSVMkiRJqinnJFUzSZIkSSphkiRJUk05J6maSZIkSVIJkyRJkmrKOUnVTJIkSZJKmCRJklRTzkmqZpIkSZJUwk6SJEk1lZ34vyoR0SciJkTEAxExJSLOWGL/BRExt+lz74i4IiKmRsQ9EbFZ077TivK/RcR+TeUjirKpEXHqG/n+2EmSJEldbR6wd2buAOwIjIiI3QAiYiiw1hL1jwZmZeaWwLnA94q6g4HDgW2AEcBFEdEWEW3Aj4D9gcHAR4u6lewkSZKkLpUNi5KiXsWWRefmHODLSxxyMDCm+Poq4L0REUX55Zk5LzP/G5gK7FpsUzPzicycD1xe1K1kJ0mSpJrK7Oi0LSKOjYhJTduxzW0pEp/7gRnAuMy8BzgBGJuZzyzR9IHAU417yIXAi8A6zeWFaUXZ0sor+XSbJElqucwcBYyq2N8O7BgR/YHfR8R7gA8De3ZKA0vYSZIkqaY6uuFikpk5OyJuBfYCtgSmNkbSWDUiphbzkKYDGwPTIqInsCbwQlP5IoOKMirKl8rhNkmS1KUiYr0iQSIi+gL7Avdm5oaZuVlmbga8UnSQAMYCI4uvPwTckplZlB9ePP22ObAVMAGYCGwVEZtHxCo0JnePXVa7TJIkSaqp7D6LSQ4AxhQTtXsAV2bmNRX1LwF+ERFTgZk0Oj1k5pSIuBJ4GFgIHF8M4xERJwA3AG3A6MycsqxGRTf6Br3OThvu3j0bJr3JPTjzya5uglRbC+dPj8683iZrb9dpv2v/MfPBTr23FcEkSZKkmuqOc5K6E+ckSZIklTBJkiSpprrrlJvuwiRJkiSphEmSJEk11WGSVMkkSZIkqYRJkiRJNZU+3VbJJEmSJKmESZIkSTXl023VTJIkSZJK2EmSJEkq4XCbJEk15WtJqpkkSZIklTBJkiSpppy4Xc0kSZIkqYRJkiRJNeVrSaqZJEmSJJUwSZIkqaack1TNJEmSJKmESZIkSTXlOknVTJIkSZJKmCRJklRTzkmqZpIkSZJUwiRJkqSacp2kaiZJkiRJJUySJEmqqfTptkomSZIkSSXsJEmSJJVwuE2SpJpy4nY1kyRJkqQSJkmSJNWUi0lWM0mSJEkqYZIkSVJNuQRANZMkSZKkEiZJkiTVlHOSqpkkSZIklTBJkiSppkySqpkkSZIklTBJkiSppsyRqpkkSZIklQjHI9UKEXFsZo7q6nZIdePPnrTimCSpVY7t6gZINeXPnrSC2EmSJEkqYSdJkiSphJ0ktYpzIqSu4c+etII4cVuSJKmESZIkSVIJO0mSJEkl7CRpuUTEiIj4W0RMjYhTS/ZvGhE3R8TkiLgtIgY17fteRDxUbId1bsullU8rft4iYu+IuK8oHxMRvnlBWgo7SXrDIqIN+BGwPzAY+GhEDF6i2veByzJze+BM4LvFse8DhgA7AsOAL0VEv05qurTSacXPW0T0AMYAh2fmtsDfgZGdcDvSSslOkpbHrsDUzHwiM+cDlwMHL1FnMHBL8fWtTfsHA3dk5sLMfBmYDIzohDZLK6tW/LytA8zPzMeKeuOAQ1t4D9JKzU6SlsdA4Kmmz9OKsmYPAB8svv4AsEZErFOUj4iIVSNiXWAvYOMWt1dambXi5+15oGdEDC2O+RD+HEpLZSdJK9qXgD0i4q/AHsB0oD0zbwSuBe4GfgP8BWjvslZKbw7L9fOWjTVfDgfOjYgJwEv4cygtlRP2tDym8/q/OgcVZYtl5tMUf9lGxOrAoZk5u9h3FnBWse/XwGNIWpqW/Lxl5l+Adxflw4G3tvImpJWZSZKWx0Rgq4jYPCJWofEX6djmChGxbjE5FOA0YHRR3lYMAxAR2wPbAzd2WsullU9Lft4iYv3in72BU4Afd8K9SCslkyS9YZm5MCJOAG4A2oDRmTklIs4EJmXmWGBP4LsRkcAdwPHF4b2AOyMCYA7wicxc2Nn3IK0sWvjzdnJEHEjjj+SLM/MWJJXytSSSJEklHG6TJEkqYSdJkiSphJ0kSZKkEnaSJEmSSthJkiRJKmEnSZIkqYSdJEmSpBJ2kqQ3kYj4ekT8LSLuiojfRMSXIuKYiJgYEQ9ExO8iYtWi7qURcXFEjI+IJyJiz4gYHRGPRMSlTeecGxHnRMSUiLgpInaNiNuKY95f1NksIu6MiPuK7Z1d9C2QpBXGTpL0JhERuwCHAjsA+wOL3vT+/zJzl8zcAXgEOLrpsLWAdwAn0XjlxbnANsB2EbFjUWc14JbM3IbGC1G/DexL463zZxZ1ZgD7ZuYQ4DDgglbcoyR1Jl9LIr157A5cnZmvAq9GxB+L8m0j4ttAf2B1Gq+5WOSPmZkR8SDwz8x8ECAipgCbAfcD84Hri/oPAvMyc0FxzGZFeS/gwqJj1Y4vTZX0JmAnSXrzuxQ4JDMfiIhP0njf1yLzin92NH296POi/39YkK+9v2hxvczsiIhFdU4C/kkjxeoBvLpib0GSOp/DbdKbx5+BgyKiT0SsDhxYlK8BPBMRvYCPt+jaawLPZGYHcASNF7JK0krNTpL0JpGZE2nMK5oMXEdjaOxF4OvAPTQ6UY+26PIXASMj4gFga+DlFl1HkjpNvJaiS1rZRcTqmTm3eILtDuDYzLyvq9slSSsj5yRJby6jImIw0AcYYwdJkv73TJIkSZJKOCdJkiSphJ0kSZKkEnaSJEmSSthJkiRJKmEnSZIkqcT/Bw3IZc5Evf7MAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "zframe = analysis.dataframe()[['config/lr', 'config/gamma', 'timesteps_total']]\n",
    "matrix = pd.pivot_table(zframe, \n",
    "                        values='timesteps_total', \n",
    "                        index=['config/lr'], \n",
    "                        columns=['config/gamma'],\n",
    "                        aggfunc=np.mean)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 10))\n",
    "sns.heatmap(matrix, annot=True, fmt='.0f')\n",
    "plt.ylabel('lr')\n",
    "plt.xlabel('gamma')\n",
    "plt.title(\"Hyperparameter matrix\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/drucev/ray_results/cartpole_test/PPO_CartPole-v1_268dd_00009_9_gamma=0.999,lr=0.0003,fcnet_hiddens=[64, 64]_2020-11-21_11-21-55'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logdir = analysis.get_best_logdir(\"timesteps_total\", mode=\"min\")\n",
    "logdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'env': 'CartPole-v1',\n",
       " 'framework': 'tfe',\n",
       " 'gamma': 0.999,\n",
       " 'lr': 0.0003,\n",
       " 'model': {'fcnet_hiddens': [64, 64]},\n",
       " 'num_gpus': 0,\n",
       " 'num_workers': 1,\n",
       " 'train_batch_size': 1000}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('%s/params.json' % logdir) as f:\n",
    "    data = json.load(f)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "== Status ==<br>Memory usage on this node: 20.0/32.0 GiB<br>Using FIFO scheduling algorithm.<br>Resources requested: 0/16 CPUs, 0/0 GPUs, 0.0/10.5 GiB heap, 0.0/3.61 GiB objects<br>Result logdir: /Users/drucev/ray_results/cartpole_test<br>Number of trials: 1/1 (1 TERMINATED)<br><table>\n",
       "<thead>\n",
       "<tr><th>Trial name                 </th><th>status    </th><th>loc  </th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">    ts</th><th style=\"text-align: right;\">  reward</th><th style=\"text-align: right;\">  episode_reward_max</th><th style=\"text-align: right;\">  episode_reward_min</th><th style=\"text-align: right;\">  episode_len_mean</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>PPO_CartPole-v1_dda4a_00000</td><td>TERMINATED</td><td>     </td><td style=\"text-align: right;\">    11</td><td style=\"text-align: right;\">         36.2623</td><td style=\"text-align: right;\">132000</td><td style=\"text-align: right;\">     500</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">                 500</td><td style=\"text-align: right;\">               500</td></tr>\n",
       "</tbody>\n",
       "</table><br><br>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-11-21 11:31:19,811\tINFO tune.py:439 -- Total run time: 51.38 seconds (50.71 seconds for the tuning loop).\n"
     ]
    }
   ],
   "source": [
    "# train some more, saving checkpoint at end\n",
    "ray.init(num_cpus=n_cpus, num_gpus=n_gpus, ignore_reinit_error=True, log_to_driver=False, dashboard_host='0.0.0.0')\n",
    "\n",
    "ppo_config = {\n",
    "    \"env\": env_name,\n",
    "    \"num_workers\": n_cpus - 1,\n",
    "    'model': {\n",
    "        'fcnet_hiddens': [16, 16]\n",
    "    },        \n",
    "    'train_batch_size': 10000,\n",
    "    \"lr\": 0.0003,\n",
    "    'gamma': 0.99,\n",
    "    'num_gpus': n_gpus  \n",
    "}\n",
    "                      \n",
    "analysis = tune.run(\n",
    "    \"PPO\",\n",
    "    name='cartpole_test',\n",
    "    verbose=1,\n",
    "\n",
    "    stop={\"episode_reward_mean\": 500},  # stop when a parameter set is able to reach 500 timesteps\n",
    "    config = ppo_config,\n",
    "    checkpoint_freq=10,\n",
    "    checkpoint_at_end=True,\n",
    "    checkpoint_score_attr='episode_reward_mean',\n",
    "    num_samples=1,  # for grid search, number of times to run each hyperparameter combo\n",
    "    #     with_server=True,\n",
    "    #     server_port=8267,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/drucev/ray_results/cartpole_test/PPO_CartPole-v1_dda4a_00000_0_2020-11-21_11-30-28'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_dir = list(analysis.trial_dataframes.keys())[0]\n",
    "results_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'_state'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-b97712e6a7dc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPPOTrainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mppo_config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0menv_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mckpoint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/opt/anaconda3/envs/test4/lib/python3.8/site-packages/ray/tune/trainable.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, checkpoint_path)\u001b[0m\n\u001b[1;32m    474\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    477\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_time_since_restore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timesteps_since_restore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/test4/lib/python3.8/site-packages/ray/rllib/agents/trainer.py\u001b[0m in \u001b[0;36mload_checkpoint\u001b[0;34m(self, checkpoint_path)\u001b[0m\n\u001b[1;32m    671\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mload_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheckpoint_path\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    672\u001b[0m         \u001b[0mextra_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"rb\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 673\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__setstate__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextra_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    674\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    675\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mDeveloperAPI\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/test4/lib/python3.8/site-packages/ray/rllib/agents/trainer_template.py\u001b[0m in \u001b[0;36m__setstate__\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;34m@\u001b[0m\u001b[0moverride\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTrainer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    163\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m__setstate__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m             \u001b[0mTrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__setstate__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    165\u001b[0m             self.train_exec_impl.shared_metrics.get().restore(\n\u001b[1;32m    166\u001b[0m                 state[\"train_exec_impl\"])\n",
      "\u001b[0;32m~/opt/anaconda3/envs/test4/lib/python3.8/site-packages/ray/rllib/agents/trainer.py\u001b[0m in \u001b[0;36m__setstate__\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m   1166\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setstate__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1167\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m\"worker\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1168\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocal_worker\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"worker\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1169\u001b[0m             \u001b[0mremote_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"worker\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1170\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremote_workers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/test4/lib/python3.8/site-packages/ray/rllib/evaluation/rollout_worker.py\u001b[0m in \u001b[0;36mrestore\u001b[0;34m(self, objs)\u001b[0m\n\u001b[1;32m    967\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msync_filters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobjs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"filters\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    968\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mobjs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"state\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 969\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicy_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    970\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    971\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mDeveloperAPI\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/envs/test4/lib/python3.8/site-packages/ray/rllib/policy/eager_tf_policy.py\u001b[0m in \u001b[0;36mset_state\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m    496\u001b[0m                     \u001b[0mopt_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    497\u001b[0m             \u001b[0;31m# Then the Policy's (NN) weights.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 498\u001b[0;31m             \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"_state\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    499\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    500\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: '_state'"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-11-21 11:40:55,268\tWARNING worker.py:1091 -- The actor or task with ID ffffffffffffffffbc4ce58d01000000 is pending and cannot currently be scheduled. It requires {CPU: 1.000000} for execution and {CPU: 1.000000} for placement, but this node only has remaining {memory: 10.693359 GiB}, {node:192.168.0.23: 1.000000}, {object_store_memory: 3.662109 GiB}. In total there are 0 pending tasks and 14 pending actors on this node. This is likely due to all cluster resources being claimed by actors. To resolve the issue, consider creating fewer actors or increase the resources available to this Ray cluster. You can ignore this message if this Ray cluster is expected to auto-scale.\n"
     ]
    }
   ],
   "source": [
    "# load last checkpoint for the runout\n",
    "# change this to latest checkpoint\n",
    "ckpoint = results_dir +  '/checkpoint_11/checkpoint-11'\n",
    "\n",
    "trainer = PPOTrainer(config=ppo_config, env=env_name)\n",
    "trainer.restore(ckpoint)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view the runout\n",
    "\n",
    "env = gym.make(env_name)\n",
    "#env.seed(GLOBAL_SEED)\n",
    "\n",
    "state = env.reset()\n",
    "trainer.compute_action(state)\n",
    "\n",
    "done = False\n",
    "\n",
    "# run an episode\n",
    "timestep = 0\n",
    "r = 0\n",
    "while not done:\n",
    "    env.render()\n",
    "    action = trainer.compute_action(state)\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    r += reward\n",
    "    timestep += 1\n",
    "print(r)\n",
    "env.close()\n",
    "timestep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}