examples/notebooks/HumanFeedback.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d8e180d",
   "metadata": {},
   "source": [
    "# Human Feedback Example"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08def965",
   "metadata": {},
   "source": [
    "## Installations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b8821b4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: prompttools in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (0.0.5)\n",
      "Requirement already satisfied: openai in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/openai-0.27.8-py3.10.egg (0.27.8)\n",
      "Requirement already satisfied: jinja2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/Jinja2-3.1.2-py3.10.egg (3.1.2)\n",
      "Requirement already satisfied: hegel in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/hegel-0.0.3-py3.10.egg (from prompttools) (0.0.3)\n",
      "Requirement already satisfied: ipywidgets in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/ipywidgets-8.0.7-py3.10.egg (from prompttools) (8.0.7)\n",
      "Requirement already satisfied: sentence-transformers in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/sentence_transformers-2.2.2-py3.10.egg (from prompttools) (2.2.2)\n",
      "Requirement already satisfied: tenacity in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/tenacity-8.2.2-py3.10.egg (from prompttools) (8.2.2)\n",
      "Requirement already satisfied: chromadb in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/chromadb-0.3.26-py3.10.egg (from prompttools) (0.3.26)\n",
      "Requirement already satisfied: jupyterlab in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/jupyterlab-4.0.2-py3.10.egg (from prompttools) (4.0.2)\n",
      "Requirement already satisfied: pandas in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/pandas-2.0.3-py3.10-macosx-10.9-x86_64.egg (from prompttools) (2.0.3)\n",
      "Requirement already satisfied: tabulate in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/tabulate-0.9.0-py3.10.egg (from prompttools) (0.9.0)\n",
      "Requirement already satisfied: requests>=2.20 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from openai) (2.28.1)\n",
      "Requirement already satisfied: tqdm in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from openai) (4.65.0)\n",
      "Requirement already satisfied: aiohttp in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from openai) (3.8.4)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jinja2) (2.1.3)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.20->openai) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.20->openai) (2022.12.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.20->openai) (1.26.15)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from requests>=2.20->openai) (2.0.4)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (23.1.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (4.0.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (1.3.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (6.0.4)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (1.3.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from aiohttp->openai) (1.9.2)\n",
      "Requirement already satisfied: pydantic>=1.9 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (2.0.2)\n",
      "Requirement already satisfied: hnswlib>=0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.7.0)\n",
      "Requirement already satisfied: clickhouse-connect>=0.5.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.6.6)\n",
      "Requirement already satisfied: duckdb>=0.7.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.8.1)\n",
      "Requirement already satisfied: fastapi>=0.85.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.100.0)\n",
      "Requirement already satisfied: uvicorn[standard]>=0.18.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.22.0)\n",
      "Requirement already satisfied: numpy>=1.21.6 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (1.25.0)\n",
      "Requirement already satisfied: posthog>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (3.0.1)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (4.7.1)\n",
      "Requirement already satisfied: pulsar-client>=3.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (3.2.0)\n",
      "Requirement already satisfied: onnxruntime>=1.14.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (1.15.1)\n",
      "Requirement already satisfied: tokenizers>=0.13.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (0.13.3)\n",
      "Requirement already satisfied: overrides>=7.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from chromadb->prompttools) (7.3.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2023.3)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pandas->prompttools) (2023.3)\n",
      "Requirement already satisfied: python-dotenv in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/python_dotenv-1.0.0-py3.10.egg (from hegel->prompttools) (1.0.0)\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (6.24.0)\n",
      "Requirement already satisfied: ipython>=6.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (8.14.0)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (5.9.0)\n",
      "Requirement already satisfied: widgetsnbextension~=4.0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (4.0.8)\n",
      "Requirement already satisfied: jupyterlab-widgets~=3.0.7 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipywidgets->prompttools) (3.0.8)\n",
      "Requirement already satisfied: async-lru>=1.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.0.3)\n",
      "Requirement already satisfied: jupyter-core in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (5.3.1)\n",
      "Requirement already satisfied: jupyter-lsp>=2.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.2.0)\n",
      "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.7.0)\n",
      "Requirement already satisfied: jupyterlab-server<3,>=2.19.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.23.0)\n",
      "Requirement already satisfied: notebook-shim>=0.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (0.2.3)\n",
      "Requirement already satisfied: packaging in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (23.0)\n",
      "Requirement already satisfied: tomli in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (2.0.1)\n",
      "Requirement already satisfied: tornado>=6.2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab->prompttools) (6.3.2)\n",
      "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (4.30.2)\n",
      "Requirement already satisfied: torch>=1.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (2.0.1)\n",
      "Requirement already satisfied: torchvision in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sentence-transformers->prompttools) (0.15.2)\n",
      "Requirement already satisfied: scikit-learn in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/scikit_learn-1.3.0-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (1.3.0)\n",
      "Requirement already satisfied: scipy in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/scipy-1.11.1-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (1.11.1)\n",
      "Requirement already satisfied: nltk in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/nltk-3.8.1-py3.10.egg (from sentence-transformers->prompttools) (3.8.1)\n",
      "Requirement already satisfied: sentencepiece in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/sentencepiece-0.1.99-py3.10-macosx-10.9-x86_64.egg (from sentence-transformers->prompttools) (0.1.99)\n",
      "Requirement already satisfied: huggingface-hub>=0.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages/huggingface_hub-0.16.4-py3.10.egg (from sentence-transformers->prompttools) (0.16.4)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: lz4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (4.3.2)\n",
      "Requirement already satisfied: zstandard in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (0.19.0)\n",
      "Requirement already satisfied: importlib-metadata in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from clickhouse-connect>=0.5.7->chromadb->prompttools) (6.8.0)\n",
      "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from fastapi>=0.85.1->chromadb->prompttools) (0.27.0)\n",
      "Requirement already satisfied: filelock in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (3.12.2)\n",
      "Requirement already satisfied: fsspec in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (2023.6.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers->prompttools) (6.0)\n",
      "Requirement already satisfied: matplotlib-inline>=0.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.6)\n",
      "Requirement already satisfied: nest-asyncio in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (1.5.6)\n",
      "Requirement already satisfied: jupyter-client>=6.1.12 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (8.3.0)\n",
      "Requirement already satisfied: comm>=0.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.3)\n",
      "Requirement already satisfied: debugpy>=1.6.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (1.6.7)\n",
      "Requirement already satisfied: psutil in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (5.9.5)\n",
      "Requirement already satisfied: pyzmq>=20 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (25.1.0)\n",
      "Requirement already satisfied: appnope in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets->prompttools) (0.1.3)\n",
      "Requirement already satisfied: stack-data in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.6.2)\n",
      "Requirement already satisfied: pickleshare in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.7.5)\n",
      "Requirement already satisfied: jedi>=0.16 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.18.2)\n",
      "Requirement already satisfied: backcall in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (0.2.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (3.0.39)\n",
      "Requirement already satisfied: pexpect>4.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (4.8.0)\n",
      "Requirement already satisfied: pygments>=2.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (2.15.1)\n",
      "Requirement already satisfied: decorator in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from ipython>=6.1.0->ipywidgets->prompttools) (5.1.1)\n",
      "Requirement already satisfied: platformdirs>=2.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-core->jupyterlab->prompttools) (3.8.1)\n",
      "Requirement already satisfied: jupyter-events>=0.6.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.6.3)\n",
      "Requirement already satisfied: anyio>=3.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (3.7.1)\n",
      "Requirement already satisfied: nbformat>=5.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (5.9.0)\n",
      "Requirement already satisfied: prometheus-client in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.17.0)\n",
      "Requirement already satisfied: websocket-client in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.6.1)\n",
      "Requirement already satisfied: send2trash in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.8.2)\n",
      "Requirement already satisfied: argon2-cffi in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (21.3.0)\n",
      "Requirement already satisfied: nbconvert>=6.4.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (7.6.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.17.1)\n",
      "Requirement already satisfied: jupyter-server-terminals in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.4.4)\n",
      "Requirement already satisfied: jsonschema>=4.17.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (4.18.0)\n",
      "Requirement already satisfied: babel>=2.10 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2.12.1)\n",
      "Requirement already satisfied: json5>=0.9.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.9.14)\n",
      "Requirement already satisfied: sympy in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (1.12)\n",
      "Requirement already satisfied: protobuf in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (4.23.4)\n",
      "Requirement already satisfied: flatbuffers in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (23.5.26)\n",
      "Requirement already satisfied: coloredlogs in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb->prompttools) (15.0.1)\n",
      "Requirement already satisfied: six>=1.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (1.16.0)\n",
      "Requirement already satisfied: monotonic>=1.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (1.6)\n",
      "Requirement already satisfied: backoff>=1.10.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->prompttools) (2.2.1)\n",
      "Requirement already satisfied: pydantic-core==2.1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pydantic>=1.9->chromadb->prompttools) (2.1.2)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pydantic>=1.9->chromadb->prompttools) (0.5.0)\n",
      "Requirement already satisfied: networkx in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers->prompttools) (3.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->prompttools) (2023.6.3)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->prompttools) (0.3.1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: click>=7.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (8.1.4)\n",
      "Requirement already satisfied: h11>=0.8 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.14.0)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.17.0)\n",
      "Requirement already satisfied: websockets>=10.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (11.0.3)\n",
      "Requirement already satisfied: httptools>=0.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.6.0)\n",
      "Requirement already satisfied: watchfiles>=0.13 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->prompttools) (0.19.0)\n",
      "Requirement already satisfied: joblib in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nltk->sentence-transformers->prompttools) (1.3.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from scikit-learn->sentence-transformers->prompttools) (3.1.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from torchvision->sentence-transformers->prompttools) (10.0.0)\n",
      "Requirement already satisfied: exceptiongroup in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.1.2)\n",
      "Requirement already satisfied: sniffio>=1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.3.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets->prompttools) (0.8.3)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.29.1)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (0.8.8)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2023.6.1)\n",
      "Requirement already satisfied: rfc3986-validator>=0.1.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.1.1)\n",
      "Requirement already satisfied: python-json-logger>=2.0.4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.0.7)\n",
      "Requirement already satisfied: rfc3339-validator in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.1.4)\n",
      "Requirement already satisfied: jupyterlab-pygments in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.2.2)\n",
      "Requirement already satisfied: bleach!=5.0.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (6.0.0)\n",
      "Requirement already satisfied: nbclient>=0.5.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.8.0)\n",
      "Requirement already satisfied: tinycss2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.2.1)\n",
      "Requirement already satisfied: defusedxml in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.7.1)\n",
      "Requirement already satisfied: mistune<4,>=2.0.3 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (3.0.1)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.5.0)\n",
      "Requirement already satisfied: beautifulsoup4 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (4.12.2)\n",
      "Requirement already satisfied: fastjsonschema in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from nbformat>=5.3.0->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.17.1)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets->prompttools) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets->prompttools) (0.2.6)\n",
      "Requirement already satisfied: argon2-cffi-bindings in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (21.2.0)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb->prompttools) (10.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from importlib-metadata->clickhouse-connect>=0.5.7->chromadb->prompttools) (3.15.0)\n",
      "Requirement already satisfied: executing>=1.2.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (1.2.0)\n",
      "Requirement already satisfied: pure-eval in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (0.2.2)\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from stack-data->ipython>=6.1.0->ipywidgets->prompttools) (2.2.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from sympy->onnxruntime>=1.14.1->chromadb->prompttools) (1.3.0)\n",
      "Requirement already satisfied: webencodings in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (0.5.1)\n",
      "Requirement already satisfied: fqdn in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.5.1)\n",
      "Requirement already satisfied: uri-template in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.3.0)\n",
      "Requirement already satisfied: jsonpointer>1.13 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (2.1)\n",
      "Requirement already satisfied: webcolors>=1.11 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.13)\n",
      "Requirement already satisfied: isoduration in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (20.11.0)\n",
      "Requirement already satisfied: cffi>=1.0.1 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (1.15.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.4.1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pycparser in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->jupyterlab->prompttools) (2.21)\n",
      "Requirement already satisfied: arrow>=0.15.0 in /Users/stevenkrawczyk/miniconda3/lib/python3.10/site-packages (from isoduration->jsonschema>=4.17.3->jupyterlab-server<3,>=2.19.0->jupyterlab->prompttools) (1.2.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install prompttools"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d598292",
   "metadata": {},
   "source": [
    "## Setup imports and API keys"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d899afc2",
   "metadata": {},
   "source": [
    "First, we'll need to set our API keys. If we are in DEBUG mode, we don't need to use real OpenAI or Hegel AI API keys, so for now we'll set them to empty strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "623cdd9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['DEBUG'] = \"\"  # Set to \"1\" if you want to use debug mode.\n",
    "os.environ['HEGELAI_API_KEY'] = \"\"  # Optional, it will be needed to use with `HegelScribe` to persist/visualize your experiments\n",
    "os.environ['OPENAI_API_KEY'] = \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "842f1e47",
   "metadata": {},
   "source": [
    "Then we'll import the relevant `prompttools` modules to setup our experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dadc080f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from prompttools.harness.prompt_template_harness import (\n",
    "    PromptTemplateExperimentationHarness,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "622dea9a",
   "metadata": {},
   "source": [
    "## Run experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3babfe5a",
   "metadata": {},
   "source": [
    "Next, we create our test inputs. For this example, we'll use a prompt template, which uses [jinja](https://jinja.palletsprojects.com/en/3.1.x/) for templating."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d878c64",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_templates = [\"Answer the following question: {{input}}\", \"Respond the following query: {{input}}\"]\n",
    "user_inputs = [{\"input\": \"Who was the first president?\"}, {\"input\": \"Who was the first president of India?\"}]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "273e9908",
   "metadata": {},
   "source": [
    "Now we can define an experimentation harness for our inputs and model. We could also pass model arguments if, for example, we wanted to change the model temperature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f16f1e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "harness = PromptTemplateExperimentationHarness(\"text-davinci-003\", prompt_templates, user_inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3fa5450",
   "metadata": {},
   "source": [
    "We can then run the experiment to get results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "866009ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "harness.run()\n",
    "harness.visualize()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be9c5ef5",
   "metadata": {},
   "source": [
    "You can use the `pivot` keyword argument to view results by the template and inputs that created them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "395c5710",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "harness.visualize(pivot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1abc866",
   "metadata": {},
   "source": [
    "## Evaluate with human feedback"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45a9900e",
   "metadata": {},
   "source": [
    "As an alternative to systematic evalaution, prompttools supports gathering human feedback directly in notebooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3defecd",
   "metadata": {},
   "outputs": [],
   "source": [
    "harness.gather_feedback()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbc46550",
   "metadata": {},
   "outputs": [],
   "source": [
    "harness.visualize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7c2fa6c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}