train_mb_offline_q_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "from base_rl.callbacks import SaveBestPolicy, SavePolicyOnTrainingEnd, SavePolicyXInterval\n",
    "from base_rl.scheduler import DecayingExpContinuousScheduler\n",
    "from base_rl.train import ParallelTrainer, OnlineTrainer\n",
    "from envs.env_creator import IBGymModelQ_creator\n",
    "from q_learning.algorithm import QLearningAlgo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = [20, 12]\n",
    "fixed_digits = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# parameters\n",
    "\n",
    "device = 'cuda'\n",
    "root_path = 'tmp'\n",
    "total_epochs = 6000\n",
    "model_name = 'model_aeq-20bits'\n",
    "initial_alpha = 0.9\n",
    "alpha_decay = 0.999\n",
    "initial_epsilon = 1.0\n",
    "epsilon_decay = 0.999\n",
    "log_dir = 'runs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps_per_episode = 1000\n",
    "gamma = 0.995\n",
    "print('alpha')\n",
    "alpha = DecayingExpContinuousScheduler(start=initial_alpha, decay=alpha_decay)\n",
    "print('epsilon')\n",
    "epsilon = DecayingExpContinuousScheduler(start=initial_epsilon, decay=epsilon_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "env_kwargs = {'steps_per_episode': steps_per_episode, 'device': device,\n",
    "              'model_path': os.path.join(root_path, 'state_quantization', model_name)}\n",
    "best_save_path = os.path.join(root_path, 'mb_q_learning', model_name, 'best_policy.pkl')\n",
    "last_save_path = os.path.join(root_path, 'mb_q_learning', model_name, 'last_save_policy.pkl')\n",
    "x_interval_save_path = os.path.join(root_path, 'mb_q_learning', model_name, 'x_interval_policy.pkl')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "algo = QLearningAlgo(\n",
    "    comment=f',type=mb_q_learning,model={model_name},gamma={gamma},total_epochs={total_epochs}',\n",
    "    epochs=total_epochs, alpha=alpha, gamma=gamma, epsilon=epsilon, env_creator=IBGymModelQ_creator,\n",
    "    env_kwargs=env_kwargs, reward_offset=2000, show_reward_type='mean', initial_q_value=0,\n",
    "    log_dir=log_dir\n",
    ")\n",
    "callbacks = [\n",
    "    SaveBestPolicy(save_path=best_save_path),\n",
    "    SavePolicyOnTrainingEnd(save_path=last_save_path),\n",
    "    SavePolicyXInterval(x_interval_save_path)\n",
    "]\n",
    "trainer = OnlineTrainer(algo=algo,callbacks=callbacks)\n",
    "trainer.fit()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}