discord evaluation prompts

.gitignore

@@ -1,3 +1,9 @@
+# autogenerated megatron config files
+megatron_config_*.json
+
+# prompt outputs
+prompts_for_summit_output*.txt
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

configs/magma_setup.yml

@@ -16,17 +16,20 @@
   ],
 
   "tokenizer-type": "HFTokenizer",
-  "vocab-file": "/gpfs/alpine/csc499/scratch/kublaikhan1/20B_tokenizer.json",
+  "vocab-file": "/gpfs/alpine/csc499/scratch/alexisroger/neox/multimodal/tokenizer.json",
 
-  "save": "/gpfs/alpine/csc499/scratch/kublaikhan1/magma/gpt-neox/checkpoints/magma_410M_testrun_0-lora-location-6",
+  "save": "/gpfs/alpine/csc499/scratch/alexisroger/neox/multimodal/models/magma_410M_testrun5_lora",
+  # "save": "/gpfs/alpine/csc499/scratch/kublaikhan1/magma/gpt-neox/checkpoints/magma_410M_testrun_0-lora-location-6",
+  "load": "/gpfs/alpine/csc499/proj-shared/edwin_runs/magma_410M_testrun05",
+  #"load": "/gpfs/alpine/csc499/proj-shared/kublaikhan1/magma_410M_testrun_0-lora-location-6/",
   #"load" : "/gpfs/alpine/csc499/scratch/kublaikhan1/magma/gpt-neox/checkpoints/magma_410M_testrun_0-lora-location-0",
-  "load": "/gpfs/alpine/csc499/proj-shared/pythia_magma_checkpoints/mp1_pp1/pythia",
+  # "load": "/gpfs/alpine/csc499/proj-shared/pythia_magma_checkpoints/mp1_pp1/pythia",
   # "load": "/gpfs/alpine/csc499/scratch/kublaikhan1/magma/gpt-neox/checkpoints/magma_410M_testrun12",
   "load_clip": "/gpfs/alpine/csc499/proj-shared/magma/models/clip",
   "checkpoint_validation_with_forward_pass": False,
 
-  "tensorboard-dir": "/gpfs/alpine/csc499/scratch/kublaikhan1/magma/gpt-neox/tensorboard/magma_410M_testrun12-6",
-  "log-dir": "/gpfs/alpine/csc499/scratch/kublaikhan1/magma/gpt-neox/logs/magma_410M_testrun_0-adapter-lora-location-6",
+  "tensorboard-dir": "/gpfs/alpine/csc499/scratch/alexisroger/neox/multimodal/tensorboard/magma_410M_testrun5_lora",
+  "log-dir": "/gpfs/alpine/csc499/scratch/alexisroger/neox/multimodal/logs/magma_410M_testrun5_lora",
   "use_wandb": False,
   "wandb_host": "https://api.wandb.ai",
   "wandb_project": "magma_410M_testrun9",
@@ -48,7 +51,7 @@
 
   # LLAMA Config
   # batch / data settings
-  "train_batch_size": 2208,
+  # "train_batch_size": 2208, ###
   "train_micro_batch_size_per_gpu": 4,
   "data-impl": "mmap",
   "split": "949,50,1",

configs/text_generation.yml

@@ -2,20 +2,20 @@
 # Make sure `load` is specified somewhere else
 {
   # Text gen type: `input-file`, `unconditional` or `interactive`
-  "text-gen-type": "unconditional",
+  "text-gen-type": "input-file",
 
   # Params for all
-  "maximum_tokens": 102,
+  "maximum_tokens": 16,
   "prompt_end": "\n",
   "temperature": 1.0,
   "top_p": 0.0,
   "top_k": 0,
   "recompute": false,
 
   # `unconditional`: samples
-  "num-samples": 10,
+  "num-samples": 1,
 
   # input/output file
-  "sample-input-file": "sample_input.txt",
-  "sample-output-file": "sample_output.txt",
+  "sample-input-file": "prompts_for_summit.txt",
+  "sample-output-file": "prompts_for_summit_output_lora_edu_05.txt",
 }

eval.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#BSUB -nnodes 7
+#BSUB -W 10
+#BSUB -q debug
+#BSUB -o /ccs/home/alexisroger/scratch/jobs/magma_pythia70m_out.%J
+#BSUB -e /ccs/home/alexisroger/scratch/jobs/magma_pythia70m_err.%J
+#BSUB -J magma_pythia70m
+#BSUB -alloc_flags gpudefault
+#BSUB -P CSC499
+
+#passage killable: 25000
+# source /gpfs/alpine/csc499/proj-shared/env_setup/setup.sh
+source /gpfs/alpine/csc499/scratch/alexisroger/neox/setup.sh
+
+#source activate /ccs/home/$(whoami)/scratch/miniconda3/envs/magma
+source activate /gpfs/alpine/csc499/scratch/alexisroger/neox/miniconda3/envs/magma
+
+# export HF_DATASETS_CACHE=/gpfs/alpine/scratch/$(whoami)/csc499/cache/hugginface
+export HF_DATASETS_CACHE=/gpfs/alpine/scratch/$(whoami)/csc499/cache/transformers
+export HUGGINGFACE_HUB_CACHE=/gpfs/alpine/scratch/$(whoami)/csc499/cache/transformers
+export HF_HOME=/gpfs/alpine/scratch/$(whoami)/csc499/cache/transformers
+export TRANSFORMERS_CACHE=/gpfs/alpine/scratch/$(whoami)/csc499/cache/transformers
+# export TRANSFORMERS_CACHE=/css/home/$(whoami)/.cache
+
+export TORCH_EXTENSIONS_DIR=/gpfs/alpine/scratch/$(whoami)/csc499/cache/torch_extensions
+
+# Write the hostfile for this job
+cat $LSB_DJOB_HOSTFILE | sort | uniq | tail -n +2 | sed -e 's/$/ slots=6/' > /ccs/home/$(whoami)/scratch/hostfiles/$LSB_JOBID-hosts
+export DLTS_HOSTFILE=/ccs/home/$(whoami)/scratch/hostfiles/$LSB_JOBID-hosts
+
+NNODES=$(wc -l < /ccs/home/$(whoami)/scratch/hostfiles/$LSB_JOBID-hosts)
+
+export WANDB_DIR=/gpfs/alpine/scratch/$(whoami)/csc499/wandb
+export WANDB_MODE=dryrun
+
+TRAIN_PATH=/gpfs/alpine/csc499/scratch/alexisroger/neox/lora
+cd $TRAIN_PATH
+
+python $TRAIN_PATH/deepy.py $TRAIN_PATH/generate.py \
+	--conf_dir $TRAIN_PATH/configs magma_pythia_410M.yml magma_setup.yml text_generation.yml

images

@@ -0,0 +1 @@
+../multimodal/images

megatron/text_generation_utils.py

@@ -607,7 +607,11 @@ def generate_samples_input_from_file(
         "generate_samples_input_from_file() loading input from {}".format(input_file)
     )
     with open(input_file, "r", encoding="utf-8") as f:
-        prompts =[json.loads(l) for l in f.readlines()]
+        prompts = [json.loads(l) for l in f.readlines()]
+        # prompts = []
+        # for l in f.readlines():
+            # print(l)
+            # prompts.append(json.loads(l))
     image_paths = [line['image'] for line in prompts] 
     texts = [line['text'] for line in prompts]