Cs674

demo.ipynb

@@ -174,30 +174,38 @@
    "execution_count": 6,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\weste\\anaconda3\\envs\\minigpt\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "iter_dt 0.00ms; iter 0: train loss 1.06407\n",
-      "iter_dt 18.17ms; iter 100: train loss 0.14712\n",
-      "iter_dt 18.70ms; iter 200: train loss 0.05315\n",
-      "iter_dt 19.65ms; iter 300: train loss 0.04404\n",
-      "iter_dt 31.64ms; iter 400: train loss 0.04724\n",
-      "iter_dt 18.43ms; iter 500: train loss 0.02521\n",
-      "iter_dt 19.83ms; iter 600: train loss 0.03352\n",
-      "iter_dt 19.58ms; iter 700: train loss 0.00539\n",
-      "iter_dt 18.72ms; iter 800: train loss 0.02057\n",
-      "iter_dt 18.26ms; iter 900: train loss 0.00360\n",
-      "iter_dt 18.50ms; iter 1000: train loss 0.00788\n",
-      "iter_dt 20.64ms; iter 1100: train loss 0.01162\n",
-      "iter_dt 18.63ms; iter 1200: train loss 0.00963\n",
-      "iter_dt 18.32ms; iter 1300: train loss 0.02066\n",
-      "iter_dt 18.40ms; iter 1400: train loss 0.01739\n",
-      "iter_dt 18.37ms; iter 1500: train loss 0.00376\n",
-      "iter_dt 18.67ms; iter 1600: train loss 0.00133\n",
-      "iter_dt 18.38ms; iter 1700: train loss 0.00179\n",
-      "iter_dt 18.66ms; iter 1800: train loss 0.00079\n",
-      "iter_dt 18.48ms; iter 1900: train loss 0.00042\n"
+      "iter_dt 0.00ms; iter 0: train loss 1.06126\n",
+      "iter_dt 13.03ms; iter 100: train loss 0.13979\n",
+      "iter_dt 14.00ms; iter 200: train loss 0.06111\n",
+      "iter_dt 22.00ms; iter 300: train loss 0.03509\n",
+      "iter_dt 17.00ms; iter 400: train loss 0.01621\n",
+      "iter_dt 15.00ms; iter 500: train loss 0.00580\n",
+      "iter_dt 13.00ms; iter 600: train loss 0.01753\n",
+      "iter_dt 14.00ms; iter 700: train loss 0.00683\n",
+      "iter_dt 14.00ms; iter 800: train loss 0.01776\n",
+      "iter_dt 12.83ms; iter 900: train loss 0.02665\n",
+      "iter_dt 14.00ms; iter 1000: train loss 0.00721\n",
+      "iter_dt 13.00ms; iter 1100: train loss 0.00226\n",
+      "iter_dt 14.00ms; iter 1200: train loss 0.02251\n",
+      "iter_dt 14.00ms; iter 1300: train loss 0.00298\n",
+      "iter_dt 14.00ms; iter 1400: train loss 0.01413\n",
+      "iter_dt 14.00ms; iter 1500: train loss 0.01876\n",
+      "iter_dt 14.03ms; iter 1600: train loss 0.00173\n",
+      "iter_dt 14.00ms; iter 1700: train loss 0.02282\n",
+      "iter_dt 14.00ms; iter 1800: train loss 0.00102\n",
+      "iter_dt 12.97ms; iter 1900: train loss 0.01723\n"
      ]
     }
    ],
@@ -317,7 +325,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.4"
+   "version": "3.11.5"
   },
   "orig_nbformat": 4,
   "vscode": {

generate.ipynb

@@ -11,7 +11,16 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\weste\\anaconda3\\envs\\minigpt\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
    "source": [
     "import torch\n",
     "from transformers import GPT2Tokenizer, GPT2LMHeadModel\n",
@@ -43,6 +52,30 @@
      "text": [
       "number of parameters: 1557.61M\n"
      ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Downloading (…)lve/main/config.json: 100%|██████████| 689/689 [00:00<00:00, 686kB/s]\n",
+      "c:\\Users\\weste\\anaconda3\\envs\\minigpt\\Lib\\site-packages\\huggingface_hub\\file_download.py:137: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\weste\\.cache\\huggingface\\hub. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
+      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
+      "  warnings.warn(message)\n",
+      "Downloading model.safetensors: 100%|██████████| 6.43G/6.43G [01:43<00:00, 62.3MB/s]\n",
+      "Downloading (…)neration_config.json: 100%|██████████| 124/124 [00:00<00:00, 124kB/s]\n"
+     ]
+    },
+    {
+     "ename": "AssertionError",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
+      "\u001b[1;32md:\\DeepLearningProjects\\minGPT\\generate.ipynb Cell 4\u001b[0m line \u001b[0;36m2\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/DeepLearningProjects/minGPT/generate.ipynb#W3sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mif\u001b[39;00m use_mingpt:\n\u001b[1;32m----> <a href='vscode-notebook-cell:/d%3A/DeepLearningProjects/minGPT/generate.ipynb#W3sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m     model \u001b[39m=\u001b[39m GPT\u001b[39m.\u001b[39;49mfrom_pretrained(model_type)\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/DeepLearningProjects/minGPT/generate.ipynb#W3sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/DeepLearningProjects/minGPT/generate.ipynb#W3sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m     model \u001b[39m=\u001b[39m GPT2LMHeadModel\u001b[39m.\u001b[39mfrom_pretrained(model_type)\n",
+      "File \u001b[1;32md:\\DeepLearningProjects\\minGPT\\mingpt\\model.py:201\u001b[0m, in \u001b[0;36mGPT.from_pretrained\u001b[1;34m(cls, model_type)\u001b[0m\n\u001b[0;32m    198\u001b[0m transposed \u001b[39m=\u001b[39m [\u001b[39m'\u001b[39m\u001b[39mattn.c_attn.weight\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mattn.c_proj.weight\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mmlp.c_fc.weight\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mmlp.c_proj.weight\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[0;32m    199\u001b[0m \u001b[39m# basically the openai checkpoints use a \"Conv1D\" module, but we only want to use a vanilla nn.Linear.\u001b[39;00m\n\u001b[0;32m    200\u001b[0m \u001b[39m# this means that we have to transpose these weights when we import them\u001b[39;00m\n\u001b[1;32m--> 201\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mlen\u001b[39m(keys) \u001b[39m==\u001b[39m \u001b[39mlen\u001b[39m(sd)\n\u001b[0;32m    202\u001b[0m \u001b[39mfor\u001b[39;00m k \u001b[39min\u001b[39;00m keys:\n\u001b[0;32m    203\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39many\u001b[39m(k\u001b[39m.\u001b[39mendswith(w) \u001b[39mfor\u001b[39;00m w \u001b[39min\u001b[39;00m transposed):\n\u001b[0;32m    204\u001b[0m         \u001b[39m# special treatment for the Conv1D weights we need to transpose\u001b[39;00m\n",
+      "\u001b[1;31mAssertionError\u001b[0m: "
+     ]
     }
    ],
    "source": [
@@ -59,7 +92,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -100,7 +133,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -152,7 +185,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.4"
+   "version": "3.11.5"
   },
   "orig_nbformat": 4,
   "vscode": {

gpt_train.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#SBATCH --time=05:10:00
+#SBATCH --mem=80G
+#SBATCH --gpus=1
+#SBATCH --nodes=1
+#SBATCH --qos=cs
+#SBATCH --partition=cs
+source min-gpt-env/bin/activate
+cd ~/minGPT/mingpt
+nvidia-smi --list-gpus
+nvidia-smi --query-gpu=memory.total --format=csv
+python jsonl_dataset.py

input.txt

@@ -0,0 +1,82 @@
+
+ChatGPT
+FADE IN:
+
+NARRATOR (V.O.)
+(ominous, echoing voice)
+In a time, where universes collide and the most unlikely of tales come to life...
+
+CUT TO:
+
+Epic scenes of the One Ring falling into a swirling vortex.
+
+NARRATOR (V.O.)
+...an unexpected hero emerges from the shadow.
+
+The vortex swirls violently and in a flash of light, Gollum lands on a street in modern day Chicago.
+
+CUT TO:
+
+Scenes of Gollum scared and overwhelmed by the tall skyscrapers, traffic, and people.
+
+NARRATOR (V.O.)
+In a world he doesn't understand...
+
+CUT TO:
+
+Gollum in an alleyway, attempting to catch a rat, startling a group of young kids.
+
+NARRATOR (V.O.)
+...he will find allies in the unlikeliest places.
+
+The kids, scared at first, gradually approach Gollum. They offer him a slice of pizza.
+
+NARRATOR (V.O.)
+...and they will teach him more than he ever expected.
+
+MONTAGE:
+
+Scenes of the kids showing Gollum around - teaching him how to cross the street, playing basketball, eating at a hot dog stand. Gollum trying and failing at modern conveniences like smartphones, vending machines, and escalators, resulting in comedic moments.
+
+CUT TO:
+
+A heartwarming scene of the kids defending Gollum from a local bully.
+
+NARRATOR (V.O.)
+Friendship. Courage. Loyalty.
+
+Gollum, seeing the kids in danger, steps up and roars at the bullies.
+
+NARRATOR (V.O.)
+In a quest to find his place in this new world...
+
+Gollum, on the top of a skyscraper, looks out at the Chicago skyline.
+
+NARRATOR (V.O.)
+...he will discover a power greater than any ring.
+
+Quick shots of exciting chase scenes, a mysterious dark force pursuing Gollum and the kids.
+
+FADE TO BLACK:
+
+Title fades in:
+
+"GOLLUM IN THE WINDY CITY"
+
+NARRATOR (V.O.)
+This summer... get ready for the adventure of a lifetime.
+
+CUT TO:
+
+Gollum with a Chicago Bulls cap, holding a basketball.
+
+GOLLUM
+(voiceover, excited)
+"Prescious... we likes this game!"
+
+The trailer ends with a comic scene of Gollum trying to dunk, but instead hanging off the hoop.
+
+NARRATOR (V.O.)
+Coming soon.
+
+FADE OUT.

mingpt/jsonl_dataset.py

@@ -0,0 +1,155 @@
+import os
+import sys
+import pandas as pd
+import matplotlib.pyplot as plt
+from pathlib import Path
+from transformers import GPT2Tokenizer
+
+
+
+import torch
+from torch.utils.data import Dataset
+
+from mingpt.model import GPT
+from mingpt.trainer import Trainer
+from mingpt.utils import set_seed, setup_logging, CfgNode as CN
+
+
+def get_config():
+
+    C = CN()
+
+    # system
+    C.system = CN()
+    C.system.seed = 3407
+    C.system.work_dir = './out/chargpt'
+
+    # data
+    C.data = JSONL_Dataset.get_default_config()
+
+    # model
+    C.model = GPT.get_default_config()
+    C.model.model_type = 'gpt-mini'
+
+    # trainer
+    C.trainer = Trainer.get_default_config()
+    C.trainer.learning_rate = 5e-4 
+    C.trainer.max_iters = 1
+    C.trainer.batch_size = 4
+
+    return C
+
+
+class JSONL_Dataset(Dataset):
+
+    @staticmethod
+    def get_default_config():
+        C = CN()
+        C.block_size = 64
+        return C
+
+    def __init__(self, file_path, block_size):
+        super().__init__()
+        self.tokenizer =  GPT2Tokenizer.from_pretrained('gpt2')
+        self.tokenizer.pad_token = self.tokenizer.eos_token
+        self.file_path = os.path.normpath(file_path)
+        self.block_size = block_size
+        self.jsonlines = []
+        with pd.read_json(self.file_path, lines=True, chunksize=1) as reader:
+            reader
+            for chunk in reader:
+                chunklet = chunk['text'].tolist()[0]
+                if isinstance(chunklet, str):
+                    self.jsonlines.append(self.tokenizer(chunklet, padding='max_length', truncation=True, max_length=self.block_size)['input_ids'])
+
+
+    def __len__(self):
+        return len(self.jsonlines)
+
+    def __getitem__(self, idx):
+        token = self.jsonlines[idx]
+        x = torch.tensor(token[:-1], dtype=torch.long)
+        y = torch.tensor(token[1:], dtype=torch.long)
+        return x, y
+
+    def get_vocab_size(self):
+        return self.tokenizer.vocab_size
+
+    def get_block_size(self):
+        return self.block_size
+
+    def text_to_token(self, str):
+        return [self.tokenizer(str, padding='max_length', truncation=True, max_length=self.block_size)['input_ids']]
+
+    def token_to_text(self, tokens):
+        return self.tokenizer.decode(tokens)
+
+
+if __name__ == "__main__":
+    #file_path = os.path.abspath("/lustre/scratch/usr/dw87/pile_data_10.jsonl")
+    file_path = os.path.join(Path().cwd(), "test_pile_file.jsonl")
+
+     # get default config and overrides from the command line, if any
+    config = get_config()
+    config.merge_from_args(sys.argv[1:])
+    print(config)
+    setup_logging(config)
+    set_seed(config.system.seed)
+
+    # construct the training dataset
+    train_dataset = JSONL_Dataset(file_path, 100)
+
+    print(train_dataset.__getitem__(0))
+
+    """
+    iter = []
+    loss = []
+
+    # construct the model
+    config.model.vocab_size = train_dataset.get_vocab_size()
+    config.model.block_size = train_dataset.get_block_size()
+    model = GPT(config.model)
+
+    # construct the trainer object
+    trainer = Trainer(config.trainer, model, train_dataset)
+
+    # iteration callback
+    def batch_end_callback(trainer):
+
+        if trainer.iter_num % 10 == 0:
+            print(f"iter_dt {trainer.iter_dt * 1000:.2f}ms; iter {trainer.iter_num}: train loss {trainer.loss.item():.5f}")
+            loss.append(trainer.loss.item())
+            iter.append(trainer.iter_num)
+
+        if trainer.iter_num % 500 == 0:
+            # evaluate both the train and test score
+            model.eval()
+            with torch.no_grad():
+                pass
+                #sample from the model...
+                #context = "o god o god"
+                #x = torch.tensor(train_dataset.text_to_token(context), dtype=torch.long).to(trainer.device)
+                #y = model.generate(x, 500, temperature=1.0, do_sample=True, top_k=10)[0]
+                #completion = ''.join(train_dataset.token_to_text(y))
+                #print(completion)
+            # save the latest model
+            print("saving model")
+            #ckpt_path = os.path.join(config.system.work_dir, "model.pt")
+            #torch.save(model.state_dict(), ckpt_path)
+            model.save('my_model.pth')
+            # revert model to training mode
+            model.train()
+
+    trainer.set_callback('on_batch_end', batch_end_callback)
+
+    # run the optimization
+    trainer.run()
+
+
+    plt.plot(iter, loss)
+    plt.xlabel('iteration')
+    plt.ylabel('loss')
+    plt.title('Training Loss')
+    plt.savefig("Loss.png")
+    """
+