YT_MPT_Instruct_7B.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1VJ7lfTYz0qu",
        "outputId": "f55c1811-87d3-41e9-8f9b-2fdaff1d83e9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m100.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m28.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m474.6/474.6 kB\u001b[0m \u001b[31m36.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m26.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m59.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.3/104.3 MB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m215.3/215.3 kB\u001b[0m \u001b[31m26.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.2/108.2 MB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip -q install git+https://github.com/huggingface/transformers # need to install from github\n",
        "!pip install -q datasets loralib sentencepiece \n",
        "!pip -q install bitsandbytes accelerate xformers einops"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hdVSk5iZ1DVB",
        "outputId": "ce652ba0-af0b-4e89-cb36-146491137310"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sat May  6 08:25:23 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  NVIDIA A100-SXM...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   32C    P0    44W / 400W |      0MiB / 40960MiB |      0%      Default |\n",
            "|                               |                      |             Disabled |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zfjJCIE5JjVI"
      },
      "source": [
        "# Mosaic Instruct MPT 7B"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import Any, Dict, Tuple\n",
        "import warnings\n",
        "\n",
        "import torch\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "from transformers import (\n",
        "    StoppingCriteria,\n",
        "    StoppingCriteriaList,\n",
        "    TextIteratorStreamer,\n",
        ")\n",
        "\n",
        "\n",
        "INSTRUCTION_KEY = \"### Instruction:\"\n",
        "RESPONSE_KEY = \"### Response:\"\n",
        "END_KEY = \"### End\"\n",
        "INTRO_BLURB = \"Below is an instruction that describes a task. Write a response that appropriately completes the request.\"\n",
        "PROMPT_FOR_GENERATION_FORMAT = \"\"\"{intro}\n",
        "{instruction_key}\n",
        "{instruction}\n",
        "{response_key}\n",
        "\"\"\".format(\n",
        "    intro=INTRO_BLURB,\n",
        "    instruction_key=INSTRUCTION_KEY,\n",
        "    instruction=\"{instruction}\",\n",
        "    response_key=RESPONSE_KEY,\n",
        ")\n",
        "\n",
        "\n",
        "class InstructionTextGenerationPipeline:\n",
        "    def __init__(\n",
        "        self,\n",
        "        model_name,\n",
        "        torch_dtype=torch.bfloat16,\n",
        "        trust_remote_code=True,\n",
        "        use_auth_token=None,\n",
        "    ) -> None:\n",
        "        self.model = AutoModelForCausalLM.from_pretrained(\n",
        "            model_name,\n",
        "            torch_dtype=torch_dtype,\n",
        "            trust_remote_code=trust_remote_code,\n",
        "            use_auth_token=use_auth_token,\n",
        "        )\n",
        "\n",
        "        tokenizer = AutoTokenizer.from_pretrained(\n",
        "            model_name,\n",
        "            trust_remote_code=trust_remote_code,\n",
        "            use_auth_token=use_auth_token,\n",
        "        )\n",
        "        if tokenizer.pad_token_id is None:\n",
        "            warnings.warn(\n",
        "                \"pad_token_id is not set for the tokenizer. Using eos_token_id as pad_token_id.\"\n",
        "            )\n",
        "            tokenizer.pad_token = tokenizer.eos_token\n",
        "        tokenizer.padding_side = \"left\"\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "        self.model.eval()\n",
        "        self.model.to(device=device, dtype=torch_dtype)\n",
        "\n",
        "        self.generate_kwargs = {\n",
        "            \"temperature\": 0.1,\n",
        "            \"top_p\": 0.92,\n",
        "            \"top_k\": 0,\n",
        "            \"max_new_tokens\": 1024,\n",
        "            \"use_cache\": True,\n",
        "            \"do_sample\": True,\n",
        "            \"eos_token_id\": self.tokenizer.eos_token_id,\n",
        "            \"pad_token_id\": self.tokenizer.pad_token_id,\n",
        "            \"repetition_penalty\": 1.1,  # 1.0 means no penalty, > 1.0 means penalty, 1.2 from CTRL paper\n",
        "        }\n",
        "\n",
        "    def format_instruction(self, instruction):\n",
        "        return PROMPT_FOR_GENERATION_FORMAT.format(instruction=instruction)\n",
        "\n",
        "    def __call__(\n",
        "        self, instruction: str, **generate_kwargs: Dict[str, Any]\n",
        "    ) -> Tuple[str, str, float]:\n",
        "        s = PROMPT_FOR_GENERATION_FORMAT.format(instruction=instruction)\n",
        "        input_ids = self.tokenizer(s, return_tensors=\"pt\").input_ids\n",
        "        input_ids = input_ids.to(self.model.device)\n",
        "        gkw = {**self.generate_kwargs, **generate_kwargs}\n",
        "        with torch.no_grad():\n",
        "            output_ids = self.model.generate(input_ids, **gkw)\n",
        "        # Slice the output_ids tensor to get only new tokens\n",
        "        new_tokens = output_ids[0, len(input_ids[0]) :]\n",
        "        output_text = self.tokenizer.decode(new_tokens, skip_special_tokens=True)\n",
        "        return output_text"
      ],
      "metadata": {
        "id": "qZ5FXdgkPFxO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Initialize the model and tokenizer\n",
        "generate = InstructionTextGenerationPipeline(\n",
        "    \"mosaicml/mpt-7b-instruct\",\n",
        "    torch_dtype=torch.bfloat16,\n",
        "    trust_remote_code=True,\n",
        ")\n",
        "stop_token_ids = generate.tokenizer.convert_tokens_to_ids([\"<|endoftext|>\"])\n",
        "\n",
        "\n",
        "# Define a custom stopping criteria\n",
        "class StopOnTokens(StoppingCriteria):\n",
        "    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:\n",
        "        for stop_id in stop_token_ids:\n",
        "            if input_ids[0][-1] == stop_id:\n",
        "                return True\n",
        "        return False\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "475c58e81703463e93aba2d54bd48027",
            "d6b2e4da04c148478d4c38a5bbd9714c",
            "bc899c26bf3d455cb1b1567afaea4e7b",
            "e3c6705338fa4a9fbf28c7d47639d8c4",
            "cb900eac4cca488fbacfdff875a8983f",
            "4834c18e62394cb8a96046b08fa20160",
            "8edf5aa218a4407daa6accca7d0a6c8b",
            "7c6667d6dc874803a04df360e474b182",
            "e306424965594ff18e9222f4ead18c4d",
            "5f524f136e794cbf91dc25c9e304f5fa",
            "55fff9c653a44ccc8ba9e8155b724265"
          ]
        },
        "id": "EBknbA-IN4aF",
        "outputId": "0be3303e-14c3-4434-a229-4e174528c962"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "475c58e81703463e93aba2d54bd48027"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-20-cf0ba811d2e7>:50: UserWarning: pad_token_id is not set for the tokenizer. Using eos_token_id as pad_token_id.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1dUMCB_kiTom"
      },
      "source": [
        "### The prompt & response"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Wo-FSysZiVkA"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import textwrap\n",
        "\n",
        "def get_prompt(instruction):\n",
        "    prompt_template = f\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n{instruction}\\n\\n### Response:\"\n",
        "    return prompt_template\n",
        "\n",
        "# print(get_prompt('What is the meaning of life?'))\n",
        "\n",
        "def parse_text(text):\n",
        "        wrapped_text = textwrap.fill(text, width=100)\n",
        "        print(wrapped_text +'\\n\\n')\n",
        "        # return assistant_text\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "prompt = 'What are the differences between alpacas, vicunas and llamas?'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6qHrby3n0109",
        "outputId": "3843b9e6-dfbb-40b4-92fe-973eb2c6b781"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Alpacas have long faces with large eyes; they can be black or brown in coloration but usually white.\n",
            "They weigh up to 100 pounds (45 kg) and stand about 3 feet tall at the shoulder. Their fleece has\n",
            "soft fibers which make it very warm for clothing use as well as other products such as pillows and\n",
            "blankets. Alpacas live on farms across South America where their fiber is harvested by shearing them\n",
            "once per year during spring time when new growth begins forming again after shedding its winter\n",
            "coat. Vicunas also come from South American countries like Peru and Chile although there are some\n",
            "populations found in North Africa too! These animals look similar to camels because of how big their\n",
            "ears are compared to body size - weighing around 150-200 lbs (70kg). The fur of these creatures\n",
            "comes in many different colors including greyish browns, blacks & whites though most commonly seen\n",
            "in shades of tan/beige due to living conditions near deserts throughout much of this region. Llamas\n",
            "share similarities with both goats & sheep since they're considered \"primitive\" mammals belonging\n",
            "within the same family group called Artiodactyla along side pigs deer etc.. However unlike either\n",
            "goat or sheep species who typically shed all hair annually through molting process known as \"blowing\n",
            "out\", llama's only need one annual grooming session instead requiring less maintenance overall than\n",
            "other types mentioned above\n",
            "\n",
            "\n",
            "CPU times: user 8.16 s, sys: 16.3 ms, total: 8.17 s\n",
            "Wall time: 8.15 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "angnwW9HG4Hv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "21126522-c1d0-484b-9cb1-87cd954e6338"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "London\n",
            "\n",
            "\n",
            "CPU times: user 63.5 ms, sys: 31 µs, total: 63.5 ms\n",
            "Wall time: 62.8 ms\n"
          ]
        }
      ],
      "source": [
        "%%time \n",
        "prompt = 'What is the capital of England?'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DGCCFto2G4Jk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e9e5f3d-fe47-4f08-92ff-0829ce40ac5e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dear Mr.Altman,  I am writing this mail with regards to your recent announcement of Open sourcing\n",
            "GPT 4 model and its training code base on GitHub under Apache 2 license. I would like to\n",
            "congratulate you for taking such bold step towards democratizing AI research by making it more\n",
            "accessible to wider community, which will help in accelerating innovation across industries as well\n",
            "as academia at large.   As someone who has been following developments around NLP space closely over\n",
            "past few years i have seen how hard it was for researchers/engineers working on similar problems to\n",
            "collaborate or build upon each other's work due to lack of access to underlying models & their\n",
            "training codes etc., so any move towards opening up these technologies can only be beneficial from\n",
            "both technical excellence perspective but also business development point of view since it helps\n",
            "foster collaboration between teams within companies  as well as outside organizations leading to\n",
            "creation of new products / services faster than ever before. Hence my recommendation would be to\n",
            "continue down this path of openness whenever possible because there are no downsides associated with\n",
            "doing so!\n",
            "\n",
            "\n",
            "CPU times: user 6.29 s, sys: 14.6 ms, total: 6.3 s\n",
            "Wall time: 6.28 s\n"
          ]
        }
      ],
      "source": [
        "%%time \n",
        "prompt = 'Write an email to Sam Altman giving reasons to open source GPT-4'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h9uswqYmG4LZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dbbfebaf-12de-4653-ccd8-e5fbb104715d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I don't have any feelings, but I can tell you what's on Wikipedia and other sources so far as it\n",
            "relates to The Simpsons.  Homer Simpson was created by Matt Groening in 1987 for his cartoon series\n",
            "called \"The Simpsons\". He has been voiced by Dan Castellaneta since 1989.\n",
            "\n",
            "\n",
            "CPU times: user 1.95 s, sys: 5.77 ms, total: 1.95 s\n",
            "Wall time: 1.95 s\n"
          ]
        }
      ],
      "source": [
        "%%time \n",
        "prompt = 'As an AI do you like the Simpsons? What do you know about Homer?'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zYM0_ryUG4NO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f287cbda-90df-4328-df2c-27b62bb483ae"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Homer Simpson was created by Matt Groening and first appeared in The Simpsons, which debuted\n",
            "December 17th 1989\n",
            "\n",
            "\n",
            "CPU times: user 732 ms, sys: 122 µs, total: 732 ms\n",
            "Wall time: 729 ms\n"
          ]
        }
      ],
      "source": [
        "%%time \n",
        "prompt = 'Tell me about Homer on the TV show the simpsons'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "prompt = 'Tell me about Homer on the TV show the simpsons in depth'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AFQ_jT0iMc4O",
        "outputId": "28ccce7b-b089-4474-9740-a7095b26b05f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Homer Simpson was created by Matt Groening and first appeared as one of many characters on The\n",
            "Tracey Ullman Show, which ran from 1987 to 1989 before being spun off into its own series called The\n",
            "Simpsons. He has been voiced since then by Dan Castellaneta who also voices other famous cartoon\n",
            "character such as Mr. Burns\n",
            "\n",
            "\n",
            "CPU times: user 2.1 s, sys: 4.83 ms, total: 2.1 s\n",
            "Wall time: 2.1 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "%%time \n",
        "prompt = 'Answer the following question by reasoning step by step. The cafeteria had 23 apples. If they used 20 for lunch, and bought 6 more, how many apple do they have?'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pmbDQ82vMPYy",
        "outputId": "f89fb8ec-ef89-468a-c85d-3c4454b74319"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The original number of apples was 23. They ate 20 from it to make lunches with them. So now there\n",
            "are only 3 left in their stockpile. To replenish what they lost, they purchased 6 new ones bringing\n",
            "their total back up to 21\n",
            "\n",
            "\n",
            "CPU times: user 1.51 s, sys: 1.82 ms, total: 1.51 s\n",
            "Wall time: 1.51 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "prompt = 'Answer the following yes\\/no question by reasoning step-by-step. \\n Can you write a whole Haiku in a single tweet?'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cHKCo6VXNByX",
        "outputId": "25c3045f-f209-4aef-fd64-b31c27c57a43"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Yes, it's possible to compose and post a haiku on Twitter using only 140 characters (including\n",
            "spaces).  The most common form of this type of poem has 5-7-5 syllable counts for each line.\n",
            "\n",
            "\n",
            "CPU times: user 1.32 s, sys: 3.82 ms, total: 1.32 s\n",
            "Wall time: 1.32 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "prompt = 'Tell me about Harry Potter and studying at Hogwarts?'\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qsn1buh6NTie",
        "outputId": "97928388-270d-4f30-d19a-9bc35906774b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Harry Potter was born to parents who were wizards, so he too has magical powers such as invisibility\n",
            "cloaking himself in mist or using his wand which can cast spells like lightning bolt, fire ball\n",
            "etc.. He lives with his aunt Petunia Dursley (his mum's sister) because of the fear by his relatives\n",
            "for him being a wizard due to the fact they are all muggle-born(non magicians). At 11 years old,\n",
            "Hagrid comes over from Hogwarts School of Witchcraft & Wizardry to take harry on a tour of the\n",
            "school but also inform petunia that she must send her son back to hogwarts immediately after their\n",
            "summer holidays end since there will be no other chance until next year! The rest of this book\n",
            "follows you through adventures when going away into your first term at Hogwarts where new friends\n",
            "are made along side learning many things throughout the 7 books\n",
            "\n",
            "\n",
            "CPU times: user 5.48 s, sys: 8.31 ms, total: 5.49 s\n",
            "Wall time: 5.47 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "prompt = \"\"\"Convert the following to JSON\n",
        "\n",
        "name: John\n",
        "age: 30\n",
        "address:\n",
        "street: 123 Main Street\n",
        "city: San Fransisco\n",
        "state: CA\n",
        "zip: 94101\n",
        "\"\"\"\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y7qsYoDUT57D",
        "outputId": "e1fdf459-f8db-418a-f3e5-0fd28326c739"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{\"name\": \"John\", \"age\":30,\"address\":{\"street\":\"123 Main street\",\"city\":\"San\n",
            "Francisco\",\"state\":\"CA\",\"zip\":94101}}\n",
            "\n",
            "\n",
            "CPU times: user 1.04 s, sys: 1.94 ms, total: 1.04 s\n",
            "Wall time: 1.04 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "prompt = \"\"\"How are you today?\"\"\"\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rp_lI2mkrIls",
        "outputId": "279e456a-de61-4962-99db-9b4de94c5334"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I am doing well, thank you for asking!\n",
            "\n",
            "\n",
            "CPU times: user 338 ms, sys: 918 µs, total: 339 ms\n",
            "Wall time: 336 ms\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "prompt = \"\"\"Write me a short plan for a 3 day trip to London\"\"\"\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AfvXtlq1rNu9",
        "outputId": "d21396a8-1998-4de9-c137-fca3bf36c48e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Day 1 - Arrive in london and check into hotel near Victoria station, have dinner at one of many\n",
            "restaurants nearby then head out to see some sites such as Big Ben or Buckingham Palace  Day 2- Head\n",
            "over to Westminster Abbey where you can tour inside and learn about British history before heading\n",
            "back towards Trafalgar Square to visit National Gallery which has free entry on Sundays! After this\n",
            "walk down Whitehall street until you reach Downing Street (the residence of the Prime Minister)\n",
            "finish off your afternoon by visiting St Paul's Cathedral with its beautiful dome overlooking the\n",
            "city from across the river Thames! Day3- Start early today so you don't miss any attractions like\n",
            "Tower Bridge, The Shard & Borough Market! Finish up your last few hours exploring Covent Garden\n",
            "market area\n",
            "\n",
            "\n",
            "CPU times: user 4.64 s, sys: 8.55 ms, total: 4.65 s\n",
            "Wall time: 4.63 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yxtk83DVyR48"
      },
      "outputs": [],
      "source": [
        "article = \"\"\"\n",
        "Content moderators under Sama, Meta’s content review sub-contractor in Africa, earlier today picketed at the company’s headquarters in Kenya demanding April salary, while urging it to observe the court orders that barred it from conducting mass layoffs.\n",
        "\n",
        "The demonstrations came after Sama, in an email, instructed moderators to clear with the company by May 11, a move the employees say is against the existing court orders.\n",
        "\n",
        "The 184 moderators sued Sama for allegedly laying them off unlawfully, after it wound down its content review arm in March, and Majorel, the social media giant’s new partner in Africa, for blacklisting on instruction by Meta.\n",
        "\n",
        "\n",
        "The court issued a temporary injunction on March 21 barring Sama from effecting any form of redundancy, and Meta from engaging Majorel, which was also instructed to refrain from blacklisting the moderators. Sama was directed to continue reviewing content on Meta’s platforms, and to be its sole provider in Africa pending determination of the case. However, Sama sent the moderators on compulsory leave in April saying it had no work for them as its contract with Meta had expired.\n",
        "\n",
        "Sama told TechCrunch that it had sent the notice “to staff whose contract had expired to go through our regular clearance process. This clearance process involves the return of company equipment to make sure that all final dues can be paid without deduction for that equipment, in accordance with Kenyan law.”\n",
        "\n",
        "It said the moderators’ contracts had ended in March after its deal with Meta expired, saying that it was only processing the moderators final dues.\n",
        "\n",
        "“We understand our former employees’ frustration because they were led by others to believe that they would all receive salary indefinitely while on leave, but that is not what the court dictated,” said Sama.\n",
        "\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "prompt = \"Please summarize this article:\\n\" + article\n",
        "generated_text = generate(prompt)\n",
        "parse_text(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1CEQDmM4xL-J",
        "outputId": "759d8176-7be2-47d3-a387-c3075fd51989"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Here’s a summary:  In early April, some content moderation workers who previously worked for Sama, a\n",
            "subcontractor of Meta (the owner of Facebook) in Africa, went on strike demanding their April\n",
            "salaries and protesting against being laid off despite a court order prohibiting such actions. The\n",
            "workers are upset that Sama has refused to follow the court order and continues to refuse to pay\n",
            "them while they remain on unpaid leave.\n",
            "\n",
            "\n",
            "CPU times: user 2.56 s, sys: 1.9 ms, total: 2.56 s\n",
            "Wall time: 2.55 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time \n",
        "prompt = \"Please extract the key info as bullet points for this article:\\n\" + article\n",
        "generated_text = generate(prompt)\n",
        "print(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zzko3r6mxU9-",
        "outputId": "941783b1-6388-497c-83c7-cdec15f1b57d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Here are some highlights from the article:  \n",
            "\n",
            "1. Content moderation workers who were previously employed by Sama, a subcontractor of Meta (Facebook), protested outside of Sama's offices in Nairobi, Kenya today, demanding their April salaries and requesting that Sama comply with court rulings prohibiting them from terminating their employment or blacklisting them. \n",
            "\n",
            "2. The protestors demanded that Sama adhere to two court decisions: one forbidding Sama from firing or blacklisting the workers, and another ordering Sama to pay the workers' outstanding wages during their mandatory unpaid leave.\n",
            "\n",
            "3. After ending its agreement with Facebook in early 2023, Sama informed the workers that their jobs had been terminated due to lack of work, even though both courts have ordered Sama to keep employing these workers until the lawsuit has been resolved.\n",
            "\n",
            "CPU times: user 5.07 s, sys: 6.27 ms, total: 5.08 s\n",
            "Wall time: 5.07 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "1K71mYsBx1r4"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "475c58e81703463e93aba2d54bd48027": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d6b2e4da04c148478d4c38a5bbd9714c",
              "IPY_MODEL_bc899c26bf3d455cb1b1567afaea4e7b",
              "IPY_MODEL_e3c6705338fa4a9fbf28c7d47639d8c4"
            ],
            "layout": "IPY_MODEL_cb900eac4cca488fbacfdff875a8983f"
          }
        },
        "d6b2e4da04c148478d4c38a5bbd9714c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4834c18e62394cb8a96046b08fa20160",
            "placeholder": "​",
            "style": "IPY_MODEL_8edf5aa218a4407daa6accca7d0a6c8b",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "bc899c26bf3d455cb1b1567afaea4e7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7c6667d6dc874803a04df360e474b182",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e306424965594ff18e9222f4ead18c4d",
            "value": 2
          }
        },
        "e3c6705338fa4a9fbf28c7d47639d8c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f524f136e794cbf91dc25c9e304f5fa",
            "placeholder": "​",
            "style": "IPY_MODEL_55fff9c653a44ccc8ba9e8155b724265",
            "value": " 2/2 [00:08&lt;00:00,  4.07s/it]"
          }
        },
        "cb900eac4cca488fbacfdff875a8983f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4834c18e62394cb8a96046b08fa20160": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8edf5aa218a4407daa6accca7d0a6c8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7c6667d6dc874803a04df360e474b182": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e306424965594ff18e9222f4ead18c4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5f524f136e794cbf91dc25c9e304f5fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55fff9c653a44ccc8ba9e8155b724265": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}