wingman.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f4c069dd-3f68-49e9-ac51-358f0591270a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-10T19:43:16.120317Z",
     "iopub.status.busy": "2024-06-10T19:43:16.119691Z",
     "iopub.status.idle": "2024-06-10T19:45:28.462193Z",
     "shell.execute_reply": "2024-06-10T19:45:28.461393Z",
     "shell.execute_reply.started": "2024-06-10T19:43:16.120294Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting mistral_inference\n",
      "  Downloading mistral_inference-1.1.0-py3-none-any.whl (21 kB)\n",
      "Collecting mistral_common\n",
      "  Downloading mistral_common-1.2.1-py3-none-any.whl (704 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m704.9/704.9 kB\u001b[0m \u001b[31m48.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.21.3)\n",
      "Collecting transformers\n",
      "  Downloading transformers-4.41.2-py3-none-any.whl (9.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.1/9.1 MB\u001b[0m \u001b[31m98.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (2.4.0)\n",
      "Collecting datasets\n",
      "  Downloading datasets-2.19.2-py3-none-any.whl (542 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.1/542.1 kB\u001b[0m \u001b[31m76.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tokenizers in /usr/local/lib/python3.9/dist-packages (0.12.1)\n",
      "Collecting tokenizers\n",
      "  Downloading tokenizers-0.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m101.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting gradio\n",
      "  Downloading gradio-4.36.1-py3-none-any.whl (12.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m75.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting fire>=0.6.0\n",
      "  Downloading fire-0.6.0.tar.gz (88 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m88.4/88.4 kB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting safetensors>=0.4.0\n",
      "  Downloading safetensors-0.4.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m91.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting simple-parsing>=0.1.5\n",
      "  Downloading simple_parsing-0.1.5-py3-none-any.whl (113 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m113.6/113.6 kB\u001b[0m \u001b[31m30.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting xformers>=0.0.24\n",
      "  Downloading xformers-0.0.26.post1-cp39-cp39-manylinux2014_x86_64.whl (222.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m222.7/222.7 MB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting typing-extensions<5.0.0,>=4.11.0\n",
      "  Downloading typing_extensions-4.12.2-py3-none-any.whl (37 kB)\n",
      "Collecting sentencepiece==0.1.99\n",
      "  Downloading sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m83.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting pydantic==2.6.1\n",
      "  Downloading pydantic-2.6.1-py3-none-any.whl (394 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m394.8/394.8 kB\u001b[0m \u001b[31m67.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting jsonschema==4.21.1\n",
      "  Downloading jsonschema-4.21.1-py3-none-any.whl (85 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.5/85.5 kB\u001b[0m \u001b[31m23.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting referencing>=0.28.4\n",
      "  Downloading referencing-0.35.1-py3-none-any.whl (26 kB)\n",
      "Collecting rpds-py>=0.7.1\n",
      "  Downloading rpds_py-0.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m95.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting attrs>=22.2.0\n",
      "  Downloading attrs-23.2.0-py3-none-any.whl (60 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting jsonschema-specifications>=2023.03.6\n",
      "  Downloading jsonschema_specifications-2023.12.1-py3-none-any.whl (18 kB)\n",
      "Collecting pydantic-core==2.16.2\n",
      "  Downloading pydantic_core-2.16.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m89.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting annotated-types>=0.4.0\n",
      "  Downloading annotated_types-0.7.0-py3-none-any.whl (13 kB)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.28.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.9.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.64.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (5.4.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.23.4)\n",
      "Collecting huggingface-hub<1.0,>=0.23.0\n",
      "  Downloading huggingface_hub-0.23.3-py3-none-any.whl (401 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.7/401.7 kB\u001b[0m \u001b[31m67.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.0)\n",
      "Collecting pyarrow>=12.0.0\n",
      "  Downloading pyarrow-16.1.0-cp39-cp39-manylinux_2_28_x86_64.whl (40.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 MB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets) (3.2.0)\n",
      "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets) (0.70.13)\n",
      "Collecting requests\n",
      "  Downloading requests-2.32.3-py3-none-any.whl (64 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.9/64.9 kB\u001b[0m \u001b[31m20.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets) (3.8.3)\n",
      "Requirement already satisfied: fsspec[http]<=2024.3.1,>=2023.1.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.1.0)\n",
      "Collecting pyarrow-hotfix\n",
      "  Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.3.5.1)\n",
      "Collecting pydub\n",
      "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
      "Collecting altair<6.0,>=4.2.0\n",
      "  Downloading altair-5.3.0-py3-none-any.whl (857 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m857.8/857.8 kB\u001b[0m \u001b[31m80.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting uvicorn>=0.14.0\n",
      "  Downloading uvicorn-0.30.1-py3-none-any.whl (62 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.4/62.4 kB\u001b[0m \u001b[31m18.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting typer<1.0,>=0.12\n",
      "  Downloading typer-0.12.3-py3-none-any.whl (47 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.2/47.2 kB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting gradio-client==1.0.1\n",
      "  Downloading gradio_client-1.0.1-py3-none-any.whl (318 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.1/318.1 kB\u001b[0m \u001b[31m57.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting aiofiles<24.0,>=22.0\n",
      "  Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.9/dist-packages (from gradio) (3.1.2)\n",
      "Collecting fastapi\n",
      "  Downloading fastapi-0.111.0-py3-none-any.whl (91 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m29.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting urllib3~=2.0\n",
      "  Downloading urllib3-2.2.1-py3-none-any.whl (121 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.1/121.1 kB\u001b[0m \u001b[31m37.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting python-multipart>=0.0.9\n",
      "  Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)\n",
      "Collecting ruff>=0.2.2\n",
      "  Downloading ruff-0.4.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.8/8.8 MB\u001b[0m \u001b[31m103.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.9/dist-packages (from gradio) (3.6.1)\n",
      "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.9/dist-packages (from gradio) (2.1.2)\n",
      "Collecting httpx>=0.24.1\n",
      "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m22.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting semantic-version~=2.0\n",
      "  Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.9/dist-packages (from gradio) (9.2.0)\n",
      "Collecting ffmpy\n",
      "  Downloading ffmpy-0.3.2.tar.gz (5.5 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting orjson~=3.0\n",
      "  Downloading orjson-3.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (142 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.5/142.5 kB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting tomlkit==0.12.0\n",
      "  Downloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\n",
      "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.9/dist-packages (from gradio) (5.10.2)\n",
      "Collecting websockets<12.0,>=10.0\n",
      "  Downloading websockets-11.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.7/129.7 kB\u001b[0m \u001b[31m32.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: toolz in /usr/local/lib/python3.9/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.12.0)\n",
      "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from fire>=0.6.0->mistral_inference) (1.14.0)\n",
      "Requirement already satisfied: termcolor in /usr/local/lib/python3.9/dist-packages (from fire>=0.6.0->mistral_inference) (2.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.3)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (4.0.2)\n",
      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.1.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.8.2)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (6.0.4)\n",
      "Collecting httpcore==1.*\n",
      "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m23.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: certifi in /usr/lib/python3/dist-packages (from httpx>=0.24.1->gradio) (2019.11.28)\n",
      "Requirement already satisfied: idna in /usr/lib/python3/dist-packages (from httpx>=0.24.1->gradio) (2.8)\n",
      "Requirement already satisfied: sniffio in /usr/local/lib/python3.9/dist-packages (from httpx>=0.24.1->gradio) (1.3.0)\n",
      "Requirement already satisfied: anyio in /usr/local/lib/python3.9/dist-packages (from httpx>=0.24.1->gradio) (3.6.2)\n",
      "Collecting h11<0.15,>=0.13\n",
      "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting huggingface-hub<1.0,>=0.23.0\n",
      "  Downloading huggingface_hub-0.23.2-py3-none-any.whl (401 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.7/401.7 kB\u001b[0m \u001b[31m64.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading huggingface_hub-0.23.1-py3-none-any.whl (401 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.3/401.3 kB\u001b[0m \u001b[31m65.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading huggingface_hub-0.23.0-py3-none-any.whl (401 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.2/401.2 kB\u001b[0m \u001b[31m62.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of httpcore to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting httpcore==1.*\n",
      "  Downloading httpcore-1.0.4-py3-none-any.whl (77 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m22.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading httpcore-1.0.3-py3-none-any.whl (77 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.0/77.0 kB\u001b[0m \u001b[31m21.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading httpcore-1.0.2-py3-none-any.whl (76 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading httpcore-1.0.1-py3-none-any.whl (76 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m21.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Downloading httpcore-1.0.0-py3-none-any.whl (76 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m21.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of httpx to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting httpx>=0.24.1\n",
      "  Downloading httpx-0.26.0-py3-none-any.whl (75 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m23.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of aiohttp to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting aiohttp\n",
      "  Downloading aiohttp-3.9.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m89.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hINFO: pip is looking at multiple versions of fsspec[http] to determine which version is compatible with other requirements. This could take a while.\n",
      "Collecting fsspec[http]<=2024.3.1,>=2023.1.0\n",
      "  Downloading fsspec-2024.3.1-py3-none-any.whl (171 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m172.0/172.0 kB\u001b[0m \u001b[31m40.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.9/dist-packages (from importlib-resources<7.0,>=1.3->gradio) (3.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.9/dist-packages (from matplotlib~=3.0->gradio) (4.38.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.9/dist-packages (from matplotlib~=3.0->gradio) (2.8.2)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib~=3.0->gradio) (1.0.7)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib~=3.0->gradio) (1.4.4)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib~=3.0->gradio) (3.0.9)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.9/dist-packages (from matplotlib~=3.0->gradio) (0.11.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n",
      "Collecting docstring-parser~=0.15\n",
      "  Downloading docstring_parser-0.16-py3-none-any.whl (36 kB)\n",
      "Requirement already satisfied: click>=8.0.0 in /usr/local/lib/python3.9/dist-packages (from typer<1.0,>=0.12->gradio) (8.1.3)\n",
      "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.9/dist-packages (from typer<1.0,>=0.12->gradio) (13.2.0)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.9/dist-packages (from typer<1.0,>=0.12->gradio) (1.5.0.post1)\n",
      "Collecting torch==2.3.0\n",
      "  Downloading torch-2.3.0-cp39-cp39-manylinux1_x86_64.whl (779.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m779.1/779.1 MB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: networkx in /usr/local/lib/python3.9/dist-packages (from torch==2.3.0->xformers>=0.0.24->mistral_inference) (3.0)\n",
      "Collecting nvidia-cuda-runtime-cu12==12.1.105\n",
      "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m83.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54\n",
      "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting sympy\n",
      "  Downloading sympy-1.12.1-py3-none-any.whl (5.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.7/5.7 MB\u001b[0m \u001b[31m110.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107\n",
      "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-nccl-cu12==2.20.5\n",
      "  Downloading nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.2/176.2 MB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1\n",
      "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26\n",
      "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106\n",
      "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m29.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106\n",
      "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cuda-nvrtc-cu12==12.1.105\n",
      "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m59.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105\n",
      "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m88.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105\n",
      "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m25.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting triton==2.3.0\n",
      "  Downloading triton-2.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (168.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m168.1/168.1 MB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting nvidia-nvjitlink-cu12\n",
      "  Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m70.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting starlette<0.38.0,>=0.37.2\n",
      "  Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m20.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1\n",
      "  Downloading ujson-5.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (53 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting fastapi-cli>=0.0.2\n",
      "  Downloading fastapi_cli-0.0.4-py3-none-any.whl (9.5 kB)\n",
      "Collecting email_validator>=2.0.0\n",
      "  Downloading email_validator-2.1.1-py3-none-any.whl (30 kB)\n",
      "Collecting dnspython>=2.0.0\n",
      "  Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m56.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: markdown-it-py<3.0.0,>=2.1.0 in /usr/local/lib/python3.9/dist-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio) (2.1.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.9/dist-packages (from rich>=10.11.0->typer<1.0,>=0.12->gradio) (2.14.0)\n",
      "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0\n",
      "  Downloading uvloop-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.5/3.5 MB\u001b[0m \u001b[31m98.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hCollecting python-dotenv>=0.13\n",
      "  Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
      "Collecting watchfiles>=0.13\n",
      "  Downloading watchfiles-0.22.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m82.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting httptools>=0.5.0\n",
      "  Downloading httptools-0.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (345 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m345.2/345.2 kB\u001b[0m \u001b[31m64.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.9/dist-packages (from markdown-it-py<3.0.0,>=2.1.0->rich>=10.11.0->typer<1.0,>=0.12->gradio) (0.1.2)\n",
      "Collecting mpmath<1.4.0,>=1.1.0\n",
      "  Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m66.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hBuilding wheels for collected packages: fire, ffmpy\n",
      "  Building wheel for fire (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for fire: filename=fire-0.6.0-py2.py3-none-any.whl size=117030 sha256=a5071f9406e39fc130227a722acd9050160561eb6091ccd16e4afd1910707313\n",
      "  Stored in directory: /root/.cache/pip/wheels/ad/22/bc/87e9046fdfb73bf7217f0a3f10fbade8a4465f2cb0e6b21bf3\n",
      "  Building wheel for ffmpy (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for ffmpy: filename=ffmpy-0.3.2-py3-none-any.whl size=5584 sha256=99658be93ab4efcb0edce8c1089a5a631590f18da1b9ce02bc09f5cde189c11d\n",
      "  Stored in directory: /root/.cache/pip/wheels/19/d7/89/d2745add714dc079c0307144c221adbc6cddbba3a0cd2a35fb\n",
      "Successfully built fire ffmpy\n",
      "Installing collected packages: sentencepiece, pydub, mpmath, ffmpy, websockets, uvloop, urllib3, ujson, typing-extensions, triton, tomlkit, sympy, semantic-version, safetensors, ruff, rpds-py, python-multipart, python-dotenv, pyarrow-hotfix, pyarrow, orjson, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, httptools, h11, fsspec, fire, docstring-parser, dnspython, attrs, annotated-types, aiofiles, watchfiles, uvicorn, starlette, simple-parsing, requests, referencing, pydantic-core, nvidia-cusparse-cu12, nvidia-cudnn-cu12, httpcore, email_validator, typer, pydantic, nvidia-cusolver-cu12, jsonschema-specifications, huggingface-hub, httpx, torch, tokenizers, jsonschema, gradio-client, fastapi-cli, datasets, xformers, transformers, mistral_common, fastapi, altair, mistral_inference, gradio\n",
      "  Attempting uninstall: sentencepiece\n",
      "    Found existing installation: sentencepiece 0.1.97\n",
      "    Uninstalling sentencepiece-0.1.97:\n",
      "      Successfully uninstalled sentencepiece-0.1.97\n",
      "  Attempting uninstall: urllib3\n",
      "    Found existing installation: urllib3 1.26.14\n",
      "    Uninstalling urllib3-1.26.14:\n",
      "      Successfully uninstalled urllib3-1.26.14\n",
      "  Attempting uninstall: typing-extensions\n",
      "    Found existing installation: typing_extensions 4.4.0\n",
      "    Uninstalling typing_extensions-4.4.0:\n",
      "      Successfully uninstalled typing_extensions-4.4.0\n",
      "  Attempting uninstall: pyarrow\n",
      "    Found existing installation: pyarrow 10.0.1\n",
      "    Uninstalling pyarrow-10.0.1:\n",
      "      Successfully uninstalled pyarrow-10.0.1\n",
      "  Attempting uninstall: fsspec\n",
      "    Found existing installation: fsspec 2023.1.0\n",
      "    Uninstalling fsspec-2023.1.0:\n",
      "      Successfully uninstalled fsspec-2023.1.0\n",
      "  Attempting uninstall: attrs\n",
      "    Found existing installation: attrs 18.2.0\n",
      "    Uninstalling attrs-18.2.0:\n",
      "      Successfully uninstalled attrs-18.2.0\n",
      "  Attempting uninstall: requests\n",
      "    Found existing installation: requests 2.28.2\n",
      "    Uninstalling requests-2.28.2:\n",
      "      Successfully uninstalled requests-2.28.2\n",
      "  Attempting uninstall: typer\n",
      "    Found existing installation: typer 0.4.2\n",
      "    Uninstalling typer-0.4.2:\n",
      "      Successfully uninstalled typer-0.4.2\n",
      "  Attempting uninstall: pydantic\n",
      "    Found existing installation: pydantic 1.9.2\n",
      "    Uninstalling pydantic-1.9.2:\n",
      "      Successfully uninstalled pydantic-1.9.2\n",
      "  Attempting uninstall: huggingface-hub\n",
      "    Found existing installation: huggingface-hub 0.12.0\n",
      "    Uninstalling huggingface-hub-0.12.0:\n",
      "      Successfully uninstalled huggingface-hub-0.12.0\n",
      "  Attempting uninstall: torch\n",
      "    Found existing installation: torch 1.12.1+cu116\n",
      "    Uninstalling torch-1.12.1+cu116:\n",
      "      Successfully uninstalled torch-1.12.1+cu116\n",
      "  Attempting uninstall: tokenizers\n",
      "    Found existing installation: tokenizers 0.12.1\n",
      "    Uninstalling tokenizers-0.12.1:\n",
      "      Successfully uninstalled tokenizers-0.12.1\n",
      "  Attempting uninstall: jsonschema\n",
      "    Found existing installation: jsonschema 4.17.3\n",
      "    Uninstalling jsonschema-4.17.3:\n",
      "      Successfully uninstalled jsonschema-4.17.3\n",
      "  Attempting uninstall: datasets\n",
      "    Found existing installation: datasets 2.4.0\n",
      "    Uninstalling datasets-2.4.0:\n",
      "      Successfully uninstalled datasets-2.4.0\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 4.21.3\n",
      "    Uninstalling transformers-4.21.3:\n",
      "      Successfully uninstalled transformers-4.21.3\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "torchvision 0.13.1+cu116 requires torch==1.12.1, but you have torch 2.3.0 which is incompatible.\n",
      "torchaudio 0.12.1+cu116 requires torch==1.12.1, but you have torch 2.3.0 which is incompatible.\n",
      "thinc 8.1.7 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.6.1 which is incompatible.\n",
      "spacy 3.4.1 requires pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4, but you have pydantic 2.6.1 which is incompatible.\n",
      "spacy 3.4.1 requires typer<0.5.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n",
      "gradient 2.0.6 requires attrs<=19, but you have attrs 23.2.0 which is incompatible.\n",
      "confection 0.0.4 requires pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4, but you have pydantic 2.6.1 which is incompatible.\n",
      "botocore 1.27.90 requires urllib3<1.27,>=1.25.4, but you have urllib3 2.2.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed aiofiles-23.2.1 altair-5.3.0 annotated-types-0.7.0 attrs-23.2.0 datasets-2.19.2 dnspython-2.6.1 docstring-parser-0.16 email_validator-2.1.1 fastapi-0.111.0 fastapi-cli-0.0.4 ffmpy-0.3.2 fire-0.6.0 fsspec-2024.3.1 gradio-4.36.1 gradio-client-1.0.1 h11-0.14.0 httpcore-1.0.5 httptools-0.6.1 httpx-0.27.0 huggingface-hub-0.23.3 jsonschema-4.21.1 jsonschema-specifications-2023.12.1 mistral_common-1.2.1 mistral_inference-1.1.0 mpmath-1.3.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105 orjson-3.10.4 pyarrow-16.1.0 pyarrow-hotfix-0.6 pydantic-2.6.1 pydantic-core-2.16.2 pydub-0.25.1 python-dotenv-1.0.1 python-multipart-0.0.9 referencing-0.35.1 requests-2.32.3 rpds-py-0.18.1 ruff-0.4.8 safetensors-0.4.3 semantic-version-2.10.0 sentencepiece-0.1.99 simple-parsing-0.1.5 starlette-0.37.2 sympy-1.12.1 tokenizers-0.19.1 tomlkit-0.12.0 torch-2.3.0 transformers-4.41.2 triton-2.3.0 typer-0.12.3 typing-extensions-4.12.2 ujson-5.10.0 urllib3-2.2.1 uvicorn-0.30.1 uvloop-0.19.0 watchfiles-0.22.0 websockets-11.0.3 xformers-0.0.26.post1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "pip install -U mistral_inference mistral_common transformers datasets tokenizers gradio accelerate \n",
    "pip install urllib3==1.26.18"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e553057b-ecd6-44ae-87b5-a5ad1a472885",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-06T18:51:15.969335Z",
     "iopub.status.busy": "2024-06-06T18:51:15.968760Z",
     "iopub.status.idle": "2024-06-06T18:53:27.915579Z",
     "shell.execute_reply": "2024-06-06T18:53:27.914802Z",
     "shell.execute_reply.started": "2024-06-06T18:51:15.969311Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ad28957ab53413da30acd344811147b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 3 files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7caefe6625b428f85702fcb044f1438",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "params.json:   0%|          | 0.00/202 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a724104dd370409dacbbaf51a6799ebc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "consolidated.safetensors:   0%|          | 0.00/44.5G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "652ebdd11d814040a1c1a6e5c45132f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model.v3:   0%|          | 0.00/588k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'/root/mistral_models/Codestral-22B-v0.1'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_path = 'mistral_models/Codestral-22B-v0.1'\n",
    "local_path.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "snapshot_download(repo_id=\"mistralai/Codestral-22B-v0.1\", allow_patterns=[\"params.json\", \"consolidated.safetensors\", \"tokenizer.model.v3\"], local_dir=mistral_models_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2bc63080-8d3a-4c3a-bf93-711cb1e16063",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-06T19:14:00.542169Z",
     "iopub.status.busy": "2024-06-06T19:14:00.541611Z",
     "iopub.status.idle": "2024-06-06T19:14:26.956982Z",
     "shell.execute_reply": "2024-06-06T19:14:26.956138Z",
     "shell.execute_reply.started": "2024-06-06T19:14:00.542144Z"
    }
   },
   "outputs": [],
   "source": [
    "from mistral_inference.model import Transformer\n",
    "from mistral_inference.generate import generate\n",
    "from mistral_common.tokens.tokenizers.mistral import MistralTokenizer\n",
    "from mistral_common.tokens.instruct.request import FIMRequest\n",
    "\n",
    "\n",
    "local_path = 'mistral_models/Codestral-22B-v0.1'\n",
    "\n",
    "tokenizer = MistralTokenizer.v3()\n",
    "model = Transformer.from_folder(local_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "ede2dab0-2d8b-4ffd-a8c6-2177e6894d73",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-06T19:31:31.402330Z",
     "iopub.status.busy": "2024-06-06T19:31:31.402048Z",
     "iopub.status.idle": "2024-06-06T19:31:32.936598Z",
     "shell.execute_reply": "2024-06-06T19:31:32.936044Z",
     "shell.execute_reply.started": "2024-06-06T19:31:31.402309Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def filter_list(list_):\n",
      "### create a function that takes a list of non-negative integers and strings and returns a new list with the strings filtered out.\n",
      "### return a new list with the strings filtered out\n",
      "    modified_list_ = []\n",
      "    for i in list_:\n",
      "        if type(i) == int:\n",
      "            modified_list_.append(i)\n",
      "    return modified_list_\n"
     ]
    }
   ],
   "source": [
    "prefix = \"\"\"def filter_list(list_):\n",
    "### create a function that takes a list of non-negative integers and strings and returns a new list with the strings filtered out.\n",
    "### return a new list with the strings filtered out\n",
    "\"\"\"\n",
    "suffix = \"\"\"    return modified_list_\"\"\"\n",
    "\n",
    "request = FIMRequest(prompt=prefix, suffix=suffix)\n",
    "\n",
    "tokens = tokenizer.encode_fim(request).tokens\n",
    "\n",
    "out_tokens, _ = generate([tokens], model, max_tokens=256, temperature=0.0, eos_id=tokenizer.instruct_tokenizer.tokenizer.eos_id)\n",
    "result = tokenizer.decode(out_tokens[0])\n",
    "\n",
    "middle = result.split(suffix)[0].strip()\n",
    "print(prefix+'    '+middle+'\\n'+suffix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "52d86217-d785-42ab-a38a-f35de0d5990b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-06T19:28:31.568253Z",
     "iopub.status.busy": "2024-06-06T19:28:31.567699Z",
     "iopub.status.idle": "2024-06-06T19:28:31.573189Z",
     "shell.execute_reply": "2024-06-06T19:28:31.572644Z",
     "shell.execute_reply.started": "2024-06-06T19:28:31.568229Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string1 = '12'\n",
    "string2 = 'eqweqw12'\n",
    "func = prefix+'    '+middle+'\\n'+suffix+'\\n'+prefix.split('def')[1][1:-1].split(':')[0]\n",
    "# print(func)\n",
    "exec(func)\n",
    "eval(prefix.split('def')[1][1:-1].split(':')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "59fed758-2000-4749-b496-e93bd31680ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-06T19:27:12.168388Z",
     "iopub.status.busy": "2024-06-06T19:27:12.167815Z",
     "iopub.status.idle": "2024-06-06T19:27:12.173148Z",
     "shell.execute_reply": "2024-06-06T19:27:12.172573Z",
     "shell.execute_reply.started": "2024-06-06T19:27:12.168362Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def same_ending(string1, string2):\n",
    "### Checks if the ordered characters in string1 are also the final ordered characters in string2. Python code:\n",
    "    if string1 == string2[-len(string1):]:\n",
    "        Bool = True\n",
    "    else:\n",
    "        Bool = False\n",
    "    return Bool\n",
    "same_ending(string1, string2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1a6a4609-baca-43d2-aec4-1dd6f1953020",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-06T19:11:54.150160Z",
     "iopub.status.busy": "2024-06-06T19:11:54.149460Z",
     "iopub.status.idle": "2024-06-06T19:11:54.154114Z",
     "shell.execute_reply": "2024-06-06T19:11:54.153686Z",
     "shell.execute_reply.started": "2024-06-06T19:11:54.150135Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def same_ending(string1, string2):\n",
    "    ### Checks if the ordered characters in string1 are also the final ordered characters in string2\n",
    "### Returns True if so, False otherwise\n",
    "    bool = False\n",
    "    if string1 == string2[-len(string1):]:\n",
    "        bool = True\n",
    "    return bool\n",
    "same_ending(string1, string2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0cb06316-e241-4396-b4de-2ae56c59f944",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-06T19:00:19.639174Z",
     "iopub.status.busy": "2024-06-06T19:00:19.638617Z",
     "iopub.status.idle": "2024-06-06T19:00:19.644325Z",
     "shell.execute_reply": "2024-06-06T19:00:19.643871Z",
     "shell.execute_reply.started": "2024-06-06T19:00:19.639150Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1,\n",
       " 1,\n",
       " 1,\n",
       " 2,\n",
       " 2,\n",
       " 3,\n",
       " 3,\n",
       " 3,\n",
       " 4,\n",
       " 5,\n",
       " 5,\n",
       " 5,\n",
       " 6,\n",
       " 6,\n",
       " 7,\n",
       " 8,\n",
       " 9,\n",
       " 9,\n",
       " 23,\n",
       " 46,\n",
       " 75,\n",
       " 87,\n",
       " 113,\n",
       " 123,\n",
       " 123,\n",
       " 213,\n",
       " 421,\n",
       " 7654,\n",
       " 134123]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr = [1,6,1,3,9,113,3,421,6,123,5,7,8,2,134123,75,123,1,2,3,4,5,7654,213,87,23,46,9,5]\n",
    "func = prefix+middle+'\\n'+suffix\n",
    "exec(func)\n",
    "bubbleSort(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6120df83-2925-4fe1-b016-1f20872155bb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-06T20:49:36.709823Z",
     "iopub.status.busy": "2024-06-06T20:49:36.709032Z",
     "iopub.status.idle": "2024-06-06T20:49:36.715652Z",
     "shell.execute_reply": "2024-06-06T20:49:36.715003Z",
     "shell.execute_reply.started": "2024-06-06T20:49:36.709798Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strin = '''\n",
    "def filter_list(list_):\n",
    "\n",
    "    modified_list_ = []\n",
    "\n",
    "    for i in list_:\n",
    "        if type(i) == int:\n",
    "            modified_list_.append(i)\n",
    "    return modified_list_'''\n",
    "exec(strin)\n",
    "eval('''filter_list([1,'1','2',2])''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f45a6996-136e-4d5f-8680-ad53d00b9c7c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-07T22:39:28.680435Z",
     "iopub.status.busy": "2024-06-07T22:39:28.679761Z",
     "iopub.status.idle": "2024-06-07T22:39:28.689158Z",
     "shell.execute_reply": "2024-06-07T22:39:28.688715Z",
     "shell.execute_reply.started": "2024-06-07T22:39:28.680410Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string1 = '''\n",
    "def same_ending(string1, string2):\n",
    "  if string1.endswith(string2):\n",
    "    return True\n",
    "  else:\n",
    "    return False\n",
    "\n",
    "# Uncomment these function calls to test your  function:\n",
    "print(same_ending(\"abc\", \"bc\"))\n",
    "\n",
    "\n",
    "'''\n",
    "x = exec(string1)\n",
    "eval('same_ending(\"abc\", \"bc\")')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38718fe6-4f4f-4e1b-aa56-f960acdff149",
   "metadata": {},
   "source": [
    "## Running the Large Language Model \n",
    "\n",
    "to get started, first paste the code in the snippet below into a Ipython Notebook cell. import your libraries yattayattayatta\n",
    "\n",
    "```\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "```\n",
    "\n",
    "load in models \n",
    "\n",
    "```\n",
    "device = \"cuda\" # the device to load the model onto\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-Instruct-v0.2\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-Instruct-v0.2\")\n",
    "```\n",
    "\n",
    "\n",
    "```from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "\n",
    "device = \"cuda\" # the device to load the model onto\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-Instruct-v0.2\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-Instruct-v0.2\")\n",
    "\n",
    "messages = [\n",
    "    {\"role\": \"user\", \"content\": \"What is your favourite condiment?\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!\"},\n",
    "    {\"role\": \"user\", \"content\": \"Do you have mayonnaise recipes?\"}\n",
    "]\n",
    "\n",
    "encodeds = tokenizer.apply_chat_template(messages, return_tensors=\"pt\")\n",
    "\n",
    "model_inputs = encodeds.to(device)\n",
    "model.to(device)\n",
    "\n",
    "generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True)\n",
    "decoded = tokenizer.batch_decode(generated_ids)\n",
    "print(decoded[0])\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5fa0a75-fa50-4bba-a2b0-64be93d628aa",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7fa32944-197e-4630-a932-10dc648bceab",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-30T23:52:55.300904Z",
     "iopub.status.busy": "2024-05-30T23:52:55.300700Z",
     "iopub.status.idle": "2024-05-30T23:52:56.245073Z",
     "shell.execute_reply": "2024-05-30T23:52:56.244507Z",
     "shell.execute_reply.started": "2024-05-30T23:52:55.300883Z"
    }
   },
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6aeb5679-6bc9-4f43-9599-8f4550976e71",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-30T23:52:56.246134Z",
     "iopub.status.busy": "2024-05-30T23:52:56.245881Z",
     "iopub.status.idle": "2024-05-30T23:53:19.017996Z",
     "shell.execute_reply": "2024-05-30T23:53:19.017523Z",
     "shell.execute_reply.started": "2024-05-30T23:52:56.246115Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac7d18abb24e4e87b4aa0e27303a973d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/3.35k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1bb519a35cf4646823e911cf7dd228e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors.index.json:   0%|          | 0.00/16.3k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "391135232f9f428682d0e7d5a280d44d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a54b65e73b604b4592e67d0ba864d499",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00001-of-00002.safetensors:   0%|          | 0.00/4.97G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "601d0a6109294dfbb0d74714aa675121",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00002-of-00002.safetensors:   0%|          | 0.00/2.67G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1881a9c0f83540f0bd0aaf29fa833129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e356dafa45af4667b3aa6ca68fbec2b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "generation_config.json:   0%|          | 0.00/172 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d51bc062cda04fc8be1caf83c07bca0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/3.17k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b2bcdac8e10400e98f0eace4ddbaed7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model:   0%|          | 0.00/500k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4af86a944cf4690b6d42b9edcc2d274",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/1.84M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9780515c4c0949c3bc642495fa1a0b20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "added_tokens.json:   0%|          | 0.00/293 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d267bbb689a24126a480fdc1b5c6f5b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/568 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "device = \"cuda\" # the device to load the model onto\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\"microsoft/Phi-3-mini-128k-instruct\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"microsoft/Phi-3-mini-128k-instruct\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3c9088c9-062d-45ca-b964-b27d0f2d7def",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-30T23:53:19.019399Z",
     "iopub.status.busy": "2024-05-30T23:53:19.019219Z",
     "iopub.status.idle": "2024-05-30T23:53:24.162833Z",
     "shell.execute_reply": "2024-05-30T23:53:24.162071Z",
     "shell.execute_reply.started": "2024-05-30T23:53:19.019381Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Phi3ForCausalLM(\n",
       "  (model): Phi3Model(\n",
       "    (embed_tokens): Embedding(32064, 3072, padding_idx=32000)\n",
       "    (embed_dropout): Dropout(p=0.0, inplace=False)\n",
       "    (layers): ModuleList(\n",
       "      (0): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (1): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (2): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (3): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (4): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (5): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (6): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (7): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (8): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (9): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (10): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (11): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (12): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (13): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (14): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (15): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (16): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (17): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (18): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (19): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (20): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (21): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (22): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (23): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (24): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (25): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (26): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (27): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (28): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (29): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (30): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "      (31): Phi3DecoderLayer(\n",
       "        (self_attn): Phi3Attention(\n",
       "          (o_proj): Linear(in_features=3072, out_features=3072, bias=False)\n",
       "          (qkv_proj): Linear(in_features=3072, out_features=9216, bias=False)\n",
       "          (rotary_emb): Phi3SuScaledRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): Phi3MLP(\n",
       "          (gate_up_proj): Linear(in_features=3072, out_features=16384, bias=False)\n",
       "          (down_proj): Linear(in_features=8192, out_features=3072, bias=False)\n",
       "          (activation_fn): SiLU()\n",
       "        )\n",
       "        (input_layernorm): Phi3RMSNorm()\n",
       "        (resid_attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (resid_mlp_dropout): Dropout(p=0.0, inplace=False)\n",
       "        (post_attention_layernorm): Phi3RMSNorm()\n",
       "      )\n",
       "    )\n",
       "    (norm): Phi3RMSNorm()\n",
       "  )\n",
       "  (lm_head): Linear(in_features=3072, out_features=32064, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages = [\n",
    "    {\"role\": \"user\", \"content\": \"What is your favourite condiment?\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!\"},\n",
    "    {\"role\": \"user\", \"content\": \"Do you have mayonnaise recipes?\"}\n",
    "]\n",
    "\n",
    "encodeds = tokenizer.apply_chat_template(messages, return_tensors=\"pt\")\n",
    "\n",
    "model_inputs = encodeds.to(device)\n",
    "model.to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ff1950d6-d5cf-411e-a4cd-ef0d8ae56c1e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-05-30T23:57:52.712312Z",
     "iopub.status.busy": "2024-05-30T23:57:52.711443Z",
     "iopub.status.idle": "2024-05-30T23:57:55.758717Z",
     "shell.execute_reply": "2024-05-30T23:57:55.757963Z",
     "shell.execute_reply.started": "2024-05-30T23:57:52.712287Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s><|user|> What is your favourite condiment?<|end|><|assistant|> Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!<|end|><|user|> Do you have mayonnaise recipes?<|end|><|assistant|> Yes, I do have a variety of mayonnaise recipes. Preparing a basic homemade mayonnaise involves slowly adding oil to an emulsified mixture of egg yolks and mustard, with seasonings. For more advanced recipes, I can suggest variations with garlic, herbs, or different types of oils.<|end|>\n"
     ]
    }
   ],
   "source": [
    "generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True)\n",
    "decoded = tokenizer.batch_decode(generated_ids)\n",
    "print(decoded[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b50784bf-9701-42c3-8aea-cb488c7bf3a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Path: Generate the text from the chat > "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}