samm_longvideos_mae_main_dev.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A jupyter notebook version file for the `main.py`\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set `autoreload` to execute the change in `.py` files.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import _pickle\n",
    "import natsort\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "from __utils__ import image_processing\n",
    "from __utils__ import label_processing\n",
    "from __utils__ import labeling\n",
    "from __utils__ import loso_preparing\n",
    "from __utils__ import functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preds_path: D:\\Databases\\SAMM_longvideos\\preds\\mae_l_swin_t_batch_size_48_epochs_25_pseudo_labeling_128.pkl\n"
     ]
    }
   ],
   "source": [
    "dataset_dir = \"D:/Databases/SAMM_longvideos\"\n",
    "# dataset_dir = \"F:/HEH/Databases/SAMM_longvideos\"\n",
    "# dataset_dir = \"/data/disk1/heh/databases/SAMM_longvideos\"\n",
    "\n",
    "images_loading = False\n",
    "image_size = 128\n",
    "load_cropped_images = False\n",
    "expression_type = \"mae\"  # macro-expression spotting\n",
    "# expression_type = \"me\"  # micro-expression spotting\n",
    "debug_preds = True\n",
    "labeling_function = \"pseudo_labeling\"\n",
    "# labeling_function = \"original_labeling\"\n",
    "model_names = {\n",
    "    0: \"SOFTNet\",\n",
    "    1: \"SOFTNetCBAM\",\n",
    "    2: \"ViT-B\",\n",
    "    3: \"SL-ViT-B\",\n",
    "    4: \"Swin-T\",\n",
    "    5: \"Swin-S\",\n",
    "    6: \"L-Swin-T\",\n",
    "    7: \"S-Swin-T\",\n",
    "    8: \"SL-Swin-T\",\n",
    "    9: \"SL-Swin-S\",\n",
    "}\n",
    "model_name = model_names[6]\n",
    "batch_size = 48\n",
    "epochs = 25\n",
    "save_preds = False\n",
    "preds_stem = (\n",
    "    f\"{expression_type}_\"\n",
    "    + model_name.lower().replace(\"-\", \"_\")\n",
    "    + f\"_batch_size_{batch_size}\"\n",
    "    + f\"_epochs_{epochs}\"\n",
    "    + f\"_{labeling_function}\"\n",
    "    + f\"_{image_size}\"\n",
    "    # + \"_2\"\n",
    ")\n",
    "preds_path = Path(dataset_dir, \"preds\", preds_stem).with_suffix(\".pkl\")\n",
    "print(f\"preds_path: {preds_path}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Images\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When debug the image processing, the videos_images is from cropped_rawpic, whereas the other variables are from rawpic.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "subject_video: 006_1\n",
      "subject_video: 006_2\n",
      "subject_video: 006_3\n",
      "subject_video: 006_4\n",
      "subject_video: 006_5\n",
      "subject_video: 006_6\n",
      "subject_video: 006_7\n",
      "subject_video: 007_3\n",
      "subject_video: 007_4\n",
      "subject_video: 007_5\n",
      "subject_video: 007_6\n",
      "subject_video: 007_7\n",
      "subject_video: 008_1\n",
      "subject_video: 008_5\n",
      "subject_video: 008_6\n",
      "subject_video: 008_7\n",
      "subject_video: 009_2\n",
      "subject_video: 009_3\n",
      "subject_video: 009_4\n",
      "subject_video: 009_6\n",
      "subject_video: 009_7\n",
      "subject_video: 010_1\n",
      "subject_video: 010_2\n",
      "subject_video: 010_3\n",
      "subject_video: 010_4\n",
      "subject_video: 010_5\n",
      "subject_video: 010_6\n",
      "subject_video: 010_7\n",
      "subject_video: 011_1\n",
      "subject_video: 011_2\n",
      "subject_video: 011_3\n",
      "subject_video: 011_4\n",
      "subject_video: 011_5\n",
      "subject_video: 011_6\n",
      "subject_video: 011_7\n",
      "subject_video: 012_3\n",
      "subject_video: 012_4\n",
      "subject_video: 012_5\n",
      "subject_video: 012_6\n",
      "subject_video: 012_7\n",
      "subject_video: 013_1\n",
      "subject_video: 013_2\n",
      "subject_video: 013_3\n",
      "subject_video: 013_6\n",
      "subject_video: 013_7\n",
      "subject_video: 014_1\n",
      "subject_video: 014_2\n",
      "subject_video: 014_3\n",
      "subject_video: 014_4\n",
      "subject_video: 014_5\n",
      "subject_video: 014_6\n",
      "subject_video: 014_7\n",
      "subject_video: 015_1\n",
      "subject_video: 015_3\n",
      "subject_video: 015_5\n",
      "subject_video: 015_6\n",
      "subject_video: 015_7\n",
      "subject_video: 016_1\n",
      "subject_video: 016_2\n",
      "subject_video: 016_4\n",
      "subject_video: 016_5\n",
      "subject_video: 016_6\n",
      "subject_video: 016_7\n",
      "subject_video: 017_1\n",
      "subject_video: 017_2\n",
      "subject_video: 017_3\n",
      "subject_video: 017_4\n",
      "subject_video: 017_5\n",
      "subject_video: 017_6\n",
      "subject_video: 018_1\n",
      "subject_video: 018_2\n",
      "subject_video: 018_3\n",
      "subject_video: 018_4\n",
      "subject_video: 018_5\n",
      "subject_video: 018_6\n",
      "subject_video: 018_7\n",
      "subject_video: 019_1\n",
      "subject_video: 019_2\n",
      "subject_video: 019_3\n",
      "subject_video: 019_4\n",
      "subject_video: 019_5\n",
      "subject_video: 019_7\n",
      "subject_video: 020_1\n",
      "subject_video: 020_2\n",
      "subject_video: 020_3\n",
      "subject_video: 020_4\n",
      "subject_video: 020_5\n",
      "subject_video: 020_6\n",
      "subject_video: 020_7\n",
      "subject_video: 021_3\n",
      "subject_video: 021_7\n",
      "subject_video: 022_2\n",
      "subject_video: 022_3\n",
      "subject_video: 022_4\n",
      "subject_video: 022_5\n",
      "subject_video: 022_6\n",
      "subject_video: 023_1\n",
      "subject_video: 023_4\n",
      "subject_video: 024_2\n",
      "subject_video: 024_3\n",
      "subject_video: 024_5\n",
      "subject_video: 025_3\n",
      "subject_video: 025_4\n",
      "subject_video: 025_5\n",
      "subject_video: 025_6\n",
      "subject_video: 026_1\n",
      "subject_video: 026_2\n",
      "subject_video: 026_3\n",
      "subject_video: 026_5\n",
      "subject_video: 026_6\n",
      "subject_video: 026_7\n",
      "subject_video: 028_4\n",
      "subject_video: 030_1\n",
      "subject_video: 030_2\n",
      "subject_video: 030_5\n",
      "subject_video: 031_3\n",
      "subject_video: 032_2\n",
      "subject_video: 032_3\n",
      "subject_video: 032_4\n",
      "subject_video: 032_5\n",
      "subject_video: 032_6\n",
      "subject_video: 033_1\n",
      "subject_video: 033_2\n",
      "subject_video: 033_3\n",
      "subject_video: 033_4\n",
      "subject_video: 033_5\n",
      "subject_video: 033_6\n",
      "subject_video: 033_7\n",
      "subject_video: 034_3\n",
      "subject_video: 034_6\n",
      "subject_video: 034_7\n",
      "subject_video: 035_1\n",
      "subject_video: 035_2\n",
      "subject_video: 035_3\n",
      "subject_video: 035_4\n",
      "subject_video: 035_5\n",
      "subject_video: 035_6\n",
      "subject_video: 035_7\n",
      "subject_video: 036_2\n",
      "subject_video: 036_4\n",
      "subject_video: 036_6\n",
      "subject_video: 036_7\n",
      "subject_video: 037_2\n",
      "subject_video: 037_3\n",
      "subject_video: 037_4\n",
      "subject_video: 037_5\n",
      "subject_video: 037_7\n"
     ]
    }
   ],
   "source": [
    "videos_images, subjects, subjects_videos_code = image_processing.load_images(\n",
    "    dataset_dir,\n",
    "    images_loading=images_loading,\n",
    "    image_size=image_size,\n",
    "    load_cropped_images=load_cropped_images,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "subjects: ['006', '007', '008', '009', '010', '011', '012', '013', '014', '015', '016', '017', '018', '019', '020', '021', '022', '023', '024', '025', '026', '028', '030', '031', '032', '033', '034', '035', '036', '037']\n",
      "subjects_videos_code: [['1', '2', '3', '4', '5', '6', '7'], ['3', '4', '5', '6', '7'], ['1', '5', '6', '7'], ['2', '3', '4', '6', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['3', '4', '5', '6', '7'], ['1', '2', '3', '6', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['1', '3', '5', '6', '7'], ['1', '2', '4', '5', '6', '7'], ['1', '2', '3', '4', '5', '6'], ['1', '2', '3', '4', '5', '6', '7'], ['1', '2', '3', '4', '5', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['3', '7'], ['2', '3', '4', '5', '6'], ['1', '4'], ['2', '3', '5'], ['3', '4', '5', '6'], ['1', '2', '3', '5', '6', '7'], ['4'], ['1', '2', '5'], ['3'], ['2', '3', '4', '5', '6'], ['1', '2', '3', '4', '5', '6', '7'], ['3', '6', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['2', '4', '6', '7'], ['2', '3', '4', '5', '7']]\n"
     ]
    }
   ],
   "source": [
    "print(\"subjects:\", subjects)\n",
    "print(\"subjects_videos_code:\", subjects_videos_code)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Excel \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Columns: Index(['subject', 'Filename', 'video_code', 'onset', 'apex', 'offset',\n",
      "       'Duration', 'expression_type', 'Action Units', 'Notes',\n",
      "       'subject_video_code', 'subject_code'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subject</th>\n",
       "      <th>Filename</th>\n",
       "      <th>video_code</th>\n",
       "      <th>onset</th>\n",
       "      <th>apex</th>\n",
       "      <th>offset</th>\n",
       "      <th>Duration</th>\n",
       "      <th>expression_type</th>\n",
       "      <th>Action Units</th>\n",
       "      <th>Notes</th>\n",
       "      <th>subject_video_code</th>\n",
       "      <th>subject_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>006</td>\n",
       "      <td>006_1_1</td>\n",
       "      <td>1</td>\n",
       "      <td>566</td>\n",
       "      <td>648</td>\n",
       "      <td>743</td>\n",
       "      <td>178</td>\n",
       "      <td>Macro</td>\n",
       "      <td>4(B/C)+7B</td>\n",
       "      <td>NaN</td>\n",
       "      <td>006_1</td>\n",
       "      <td>006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>006</td>\n",
       "      <td>006_1_2</td>\n",
       "      <td>1</td>\n",
       "      <td>3562</td>\n",
       "      <td>3588</td>\n",
       "      <td>3632</td>\n",
       "      <td>71</td>\n",
       "      <td>Micro - 1/2</td>\n",
       "      <td>4+7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>006_1</td>\n",
       "      <td>006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>006</td>\n",
       "      <td>006_1_3</td>\n",
       "      <td>1</td>\n",
       "      <td>1912</td>\n",
       "      <td>1948</td>\n",
       "      <td>1988</td>\n",
       "      <td>77</td>\n",
       "      <td>Micro - 1/2</td>\n",
       "      <td>4</td>\n",
       "      <td>While blinking</td>\n",
       "      <td>006_1</td>\n",
       "      <td>006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>006</td>\n",
       "      <td>006_1_4</td>\n",
       "      <td>1</td>\n",
       "      <td>324</td>\n",
       "      <td>368</td>\n",
       "      <td>403</td>\n",
       "      <td>80</td>\n",
       "      <td>Micro - 1/2</td>\n",
       "      <td>4+7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>006_1</td>\n",
       "      <td>006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>006</td>\n",
       "      <td>006_1_5</td>\n",
       "      <td>1</td>\n",
       "      <td>3343</td>\n",
       "      <td>3388</td>\n",
       "      <td>3424</td>\n",
       "      <td>82</td>\n",
       "      <td>Micro - 1/2</td>\n",
       "      <td>4+7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>006_1</td>\n",
       "      <td>006</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  subject Filename video_code onset  apex offset Duration expression_type   \n",
       "0     006  006_1_1          1   566   648    743      178           Macro  \\\n",
       "1     006  006_1_2          1  3562  3588   3632       71     Micro - 1/2   \n",
       "2     006  006_1_3          1  1912  1948   1988       77     Micro - 1/2   \n",
       "3     006  006_1_4          1   324   368    403       80     Micro - 1/2   \n",
       "4     006  006_1_5          1  3343  3388   3424       82     Micro - 1/2   \n",
       "\n",
       "  Action Units           Notes subject_video_code subject_code  \n",
       "0    4(B/C)+7B             NaN              006_1          006  \n",
       "1          4+7             NaN              006_1          006  \n",
       "2            4  While blinking              006_1          006  \n",
       "3          4+7             NaN              006_1          006  \n",
       "4          4+7             NaN              006_1          006  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Excel_data = label_processing.load_excel(dataset_dir)\n",
    "Excel_data.head(5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Ground Truth Labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "required_videos_index:  [0, 1, 2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 70, 71, 72, 73, 74, 75, 76, 77, 78, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 91, 92, 95, 97, 98, 99, 100, 101, 103, 106, 107, 110, 111, 112, 113, 114, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 129, 130, 131, 132, 133, 134, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146]\n",
      "len(clean_videos_images) = 130\n"
     ]
    }
   ],
   "source": [
    "(\n",
    "    clean_videos_images,\n",
    "    clean_subjects_videos_code,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    ") = label_processing.load_ground_truth_labels(\n",
    "    dataset_dir,\n",
    "    expression_type,\n",
    "    videos_images,\n",
    "    subjects_videos_code,\n",
    "    subjects,\n",
    "    Excel_data,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(clean_subjects):  29\n",
      "clean_subjects:  ['006' '007' '008' '009' '010' '011' '012' '013' '014' '015' '016' '017'\n",
      " '018' '019' '020' '021' '022' '023' '024' '025' '026' '028' '030' '032'\n",
      " '033' '034' '035' '036' '037']\n",
      "len(clean_subjects_videos_code):  29\n",
      "clean_subjects_videos_codes:  [['1', '2', '3', '4', '5', '6', '7'], ['5', '6', '7'], ['1', '5', '6', '7'], ['3', '4', '6', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['3', '4', '5', '6', '7'], ['1', '2', '3', '6', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['1', '3', '5', '6', '7'], ['1', '2', '4', '5', '6', '7'], ['1', '2', '3', '4', '5', '6'], ['2', '3', '4', '5', '6', '7'], ['1', '2', '3', '5', '7'], ['1', '2', '3', '4', '5', '6', '7'], ['3'], ['2', '3', '6'], ['4'], ['2', '3', '5'], ['3', '5'], ['2', '3', '7'], ['4'], ['1', '2', '5'], ['2', '3', '4', '5', '6'], ['1', '2', '3', '4', '5', '6', '7'], ['6', '7'], ['1', '2', '3', '4', '6', '7'], ['2', '4', '6', '7'], ['2', '3', '4', '5', '7']]\n",
      "len(clean_subjects_videos_ground_truth_labels):  29\n",
      "clean_subjects_videos_ground_truth_labels:  [[[[565, 742], [8606, 8727]], [[326, 814], [1658, 2261]], [[721, 1155], [2280, 2905], [3003, 3411]], [[4203, 4379], [118, 1658], [1726, 4238], [4484, 4708]], [[104, 401], [3336, 3456], [4266, 4432], [744, 1132], [1384, 1728], [1996, 2304]], [[495, 730], [519, 1023], [1684, 2131], [3913, 4585]], [[193, 430], [544, 708]]], [[[2650, 3459]], [[1196, 1466]], [[595, 1043], [1523, 1834], [6755, 7284]]], [[[301, 668]], [[741, 912]], [[413, 549]], [[704, 1076], [5044, 5280]]], [[[2441, 2722], [1492, 1604], [413, 540]], [[563, 865]], [[461, 594]], [[102, 274]]], [[[837, 1134], [88, 208]], [[1377, 1493], [736, 914], [1672, 1962], [3392, 3762], [4243, 4421], [4433, 4643], [4644, 4885], [2404, 2516]], [[792, 1484], [1366, 1479], [1999, 2424], [3777, 3934]], [[1000, 1179]], [[664, 1635], [2424, 2871]], [[660, 846]], [[126, 625]]], [[[31, 306], [4023, 4346], [5736, 5913]], [[3728, 3842], [4233, 4364]], [[913, 1069], [1454, 1780], [835, 955]], [[4870, 5040], [6677, 6820], [2968, 3075], [62, 284], [313, 744], [802, 1014], [1639, 1829], [1902, 2069], [2476, 2674], [2691, 2829]], [[8660, 8861], [1019, 1121], [6739, 6865], [180, 621], [833, 1027], [1777, 1949], [2450, 2587], [3667, 3817], [4048, 4229], [4460, 4645], [6862, 7188]], [[4170, 4338], [4770, 5719], [178, 442], [555, 816], [967, 2050], [2683, 2826], [2856, 3987]], [[1203, 1312], [3584, 3704], [350, 501], [576, 1178], [2841, 3133], [4119, 4347], [4351, 4556], [5570, 5815], [5902, 6252], [7935, 8405]]], [[[5029, 5474]], [[1340, 1524]], [[616, 800]], [[2887, 3066], [4083, 4279], [4342, 4514], [238, 353]], [[951, 1180]]], [[[7261, 7371], [676, 832], [3698, 3902], [7183, 7364]], [[443, 602], [2177, 2346]], [[636, 774]], [[392, 696]], [[223, 422], [1828, 1980], [3254, 3407], [4040, 4271], [7533, 7829], [8214, 8569]]], [[[599, 940], [1501, 1699]], [[753, 1270], [2385, 2590]], [[410, 565], [1925, 2066], [1964, 2071]], [[342, 547], [1503, 1736], [2900, 3093]], [[444, 612], [1808, 1921]], [[4641, 4913], [1169, 1274], [2091, 2210]], [[440, 582], [2327, 2791], [3015, 3439], [4723, 4994], [6693, 7114]]], [[[22, 167], [296, 594], [4853, 5167]], [[344, 706], [205, 307]], [[424, 616], [1050, 1202], [875, 978]], [[338, 549], [530, 906], [2437, 2777], [4561, 4943], [5586, 5889]], [[68, 437], [438, 801]]], [[[501, 742], [2524, 2860]], [[782, 1059]], [[690, 1013]], [[38, 183]], [[349, 2144], [2375, 2633]], [[116, 562], [2493, 2685], [3290, 3463], [2798, 2907]]], [[[651, 1049]], [[458, 591]], [[314, 432]], [[207, 441]], [[525, 1007]], [[1599, 1891]]], [[[242, 813], [1964, 2157]], [[1500, 1681]], [[417, 540], [391, 548], [536, 784], [1473, 1888], [1891, 2055], [4774, 5130]], [[442, 740], [2040, 2671], [5427, 5578]], [[890, 1343], [4333, 4704], [4717, 5191]], [[138, 499]]], [[[454, 673], [722, 860], [3703, 3857], [3986, 4157], [4270, 4524], [5485, 5874]], [[43, 282]], [[2786, 2887]], [[449, 615], [1187, 1329], [2093, 2232], [2138, 2239]], [[397, 531], [974, 1139]]], [[[2200, 2346], [6674, 6819], [573, 682]], [[664, 802]], [[628, 2509], [2942, 3600]], [[583, 727], [1079, 1235], [2833, 3005]], [[548, 706]], [[208, 1240], [1309, 2261], [3230, 3476], [3531, 5000]], [[9204, 9636], [88, 264], [2546, 2694], [2853, 3310], [4758, 4949], [5212, 5518], [7710, 7856], [8430, 8574]]], [[[883, 1487], [3050, 3337]]], [[[1246, 1417], [2758, 2984], [2053, 2176], [5038, 5199]], [[477, 930], [1440, 1561]], [[496, 639]]], [[[150, 601]]], [[[760, 874], [563, 719]], [[2046, 2233], [2270, 2483], [2714, 2832], [1859, 2015]], [[155, 269]]], [[[543, 770], [2394, 2577], [984, 1101], [3034, 3183]], [[839, 978]]], [[[2048, 2149], [723, 835], [1238, 1352]], [[3861, 4131], [3640, 3860], [724, 861], [1340, 1482]], [[817, 1028]]], [[[1595, 1833], [938, 1065]]], [[[2753, 3309], [4275, 4494]], [[941, 1136]], [[375, 700], [2113, 2437]]], [[[583, 874], [1445, 1795], [2192, 2904], [2914, 3463], [4572, 4712]], [[229, 399]], [[4586, 4783], [4880, 5106]], [[814, 955]], [[451, 888]]], [[[2208, 2942], [2949, 5687], [871, 1212]], [[2805, 2910], [2090, 2314]], [[616, 746]], [[538, 821]], [[391, 528]], [[854, 1218]], [[4052, 4502], [159, 324]]], [[[3521, 3710], [207, 365]], [[1037, 1543], [2126, 2342]]], [[[522, 729]], [[1642, 1755], [526, 741], [1793, 1943]], [[1264, 1537], [264, 828]], [[1463, 1590]], [[3387, 5018], [5328, 5632], [4386, 4849]], [[4526, 4741], [6963, 7153], [5190, 5293]]], [[[1227, 3176], [3310, 3858], [1743, 2121], [2638, 3169]], [[2931, 3110], [3111, 3469], [1023, 1311]], [[1030, 1670], [1671, 2448], [2449, 3207], [3208, 5456]], [[1660, 5595], [2208, 2739], [5872, 8200], [1051, 1449], [3414, 3718], [6009, 6313]]], [[[1170, 1306]], [[1479, 1741], [3303, 3471]], [[1486, 1604], [4987, 5112], [3403, 3972]], [[1716, 1842], [4206, 4655]], [[803, 1132]]]]\n"
     ]
    }
   ],
   "source": [
    "print(\"len(clean_subjects): \", len(clean_subjects))\n",
    "print(\"clean_subjects: \", clean_subjects)\n",
    "print(\"len(clean_subjects_videos_code): \", len(clean_subjects_videos_code))\n",
    "print(\"clean_subjects_videos_codes: \", clean_subjects_videos_code)\n",
    "print(\n",
    "    \"len(clean_subjects_videos_ground_truth_labels): \",\n",
    "    len(clean_subjects_videos_ground_truth_labels),\n",
    ")\n",
    "print(\n",
    "    \"clean_subjects_videos_ground_truth_labels: \",\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate `k`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k (Half of average length of expression) =  169\n"
     ]
    }
   ],
   "source": [
    "k = label_processing.calculate_k(clean_subjects_videos_ground_truth_labels)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Labeling\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total frames: 438311\n"
     ]
    }
   ],
   "source": [
    "if labeling_function == \"pseudo_labeling\":\n",
    "    labels = labeling.get_pseudo_labels(\n",
    "        clean_videos_images, clean_subjects_videos_ground_truth_labels, k\n",
    "    )\n",
    "elif labeling_function == \"original_labeling\":\n",
    "    labels = labeling.get_original_labels(\n",
    "        clean_videos_images, clean_subjects_videos_ground_truth_labels, k\n",
    "    )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare for LOSO\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frame Index for each subject:-\n",
      "\n",
      "subject 006 ( group = 0): 0 -> 29116\n",
      "subject 006 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  7\n",
      "\n",
      "subject 007 ( group = 1): 29116 -> 42609\n",
      "subject 007 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  10\n",
      "\n",
      "subject 008 ( group = 2): 42609 -> 50824\n",
      "subject 008 has 4 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  14\n",
      "\n",
      "subject 009 ( group = 3): 50824 -> 55844\n",
      "subject 009 has 4 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  18\n",
      "\n",
      "subject 010 ( group = 4): 55844 -> 73934\n",
      "subject 010 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  25\n",
      "\n",
      "subject 011 ( group = 5): 73934 -> 116503\n",
      "subject 011 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  32\n",
      "\n",
      "subject 012 ( group = 6): 116503 -> 130313\n",
      "subject 012 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  37\n",
      "\n",
      "subject 013 ( group = 7): 130313 -> 151145\n",
      "subject 013 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  42\n",
      "\n",
      "subject 014 ( group = 8): 151145 -> 178939\n",
      "subject 014 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  49\n",
      "\n",
      "subject 015 ( group = 9): 178939 -> 193231\n",
      "subject 015 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  54\n",
      "\n",
      "subject 016 ( group = 10): 193231 -> 213712\n",
      "subject 016 has 6 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  60\n",
      "\n",
      "subject 017 ( group = 11): 213712 -> 222201\n",
      "subject 017 has 6 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  66\n",
      "\n",
      "subject 018 ( group = 12): 222201 -> 244739\n",
      "subject 018 has 6 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  72\n",
      "\n",
      "subject 019 ( group = 13): 244739 -> 258895\n",
      "subject 019 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  77\n",
      "\n",
      "subject 020 ( group = 14): 258895 -> 289954\n",
      "subject 020 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  84\n",
      "\n",
      "subject 021 ( group = 15): 289954 -> 293204\n",
      "subject 021 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  85\n",
      "\n",
      "subject 022 ( group = 16): 293204 -> 302931\n",
      "subject 022 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  88\n",
      "\n",
      "subject 023 ( group = 17): 302931 -> 303763\n",
      "subject 023 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  89\n",
      "\n",
      "subject 024 ( group = 18): 303763 -> 308959\n",
      "subject 024 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  92\n",
      "\n",
      "subject 025 ( group = 19): 308959 -> 314122\n",
      "subject 025 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  94\n",
      "\n",
      "subject 026 ( group = 20): 314122 -> 327510\n",
      "subject 026 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  97\n",
      "\n",
      "subject 028 ( group = 21): 327510 -> 329342\n",
      "subject 028 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  98\n",
      "\n",
      "subject 030 ( group = 22): 329342 -> 339336\n",
      "subject 030 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  101\n",
      "\n",
      "subject 032 ( group = 23): 339336 -> 354127\n",
      "subject 032 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  106\n",
      "\n",
      "subject 033 ( group = 24): 354127 -> 372229\n",
      "subject 033 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  113\n",
      "\n",
      "subject 034 ( group = 25): 372229 -> 379892\n",
      "subject 034 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  115\n",
      "\n",
      "subject 035 ( group = 26): 379892 -> 401746\n",
      "subject 035 has 6 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  121\n",
      "\n",
      "subject 036 ( group = 27): 401746 -> 422588\n",
      "subject 036 has 4 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  125\n",
      "\n",
      "subject 037 ( group = 28): 422588 -> 438311\n",
      "subject 037 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  130\n"
     ]
    }
   ],
   "source": [
    "y, groups = loso_preparing.prepare_for_loso(\n",
    "    labels,\n",
    "    clean_subjects,\n",
    "    clean_videos_images,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    k,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "if debug_preds is False:\n",
    "    from __utils__.training_dev import train\n",
    "\n",
    "    preds = train(\n",
    "        dataset_dir,\n",
    "        clean_subjects,\n",
    "        image_size=image_size,\n",
    "        y=y,\n",
    "        expression_type=expression_type,\n",
    "        model_name=model_name,\n",
    "        train_or_not=True,\n",
    "        batch_size=batch_size,\n",
    "        epochs=epochs,\n",
    "        preds_path=preds_path,\n",
    "    )\n",
    "else:\n",
    "    with open(preds_path, \"rb\") as pkl_file:\n",
    "        preds = _pickle.load(pkl_file)\n",
    "        pkl_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "if save_preds is True:\n",
    "    with open(preds_path, \"wb\") as pkl_file:\n",
    "        _pickle.dump(preds, pkl_file)\n",
    "        pkl_file.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spotting and Evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1/29 is in process.\n",
      "0 video(s) have been processed.\n",
      "The current video be processed: subject 006, video 1\n",
      "The current video be processed: subject 006, video 2\n",
      "The current video be processed: subject 006, video 3\n",
      "The current video be processed: subject 006, video 4\n",
      "The current video be processed: subject 006, video 5\n",
      "The current video be processed: subject 006, video 6\n",
      "The current video be processed: subject 006, video 7\n",
      "\n",
      "True Positive: 4, False Posive: 12, False Negative: 19\n",
      "Precision = 0.2500000000000002, Recall =0.1739130434782611, F1-Score = 0.2051282051282056\n",
      "Split 1/29 is processed.\n",
      "\n",
      "Split 2/29 is in process.\n",
      "7 video(s) have been processed.\n",
      "The current video be processed: subject 007, video 5\n",
      "The current video be processed: subject 007, video 6\n",
      "The current video be processed: subject 007, video 7\n",
      "\n",
      "True Positive: 5, False Posive: 15, False Negative: 23\n",
      "Precision = 0.2500000000000002, Recall =0.1785714285714288, F1-Score = 0.2083333333333338\n",
      "Split 2/29 is processed.\n",
      "\n",
      "Split 3/29 is in process.\n",
      "10 video(s) have been processed.\n",
      "The current video be processed: subject 008, video 1\n",
      "The current video be processed: subject 008, video 5\n",
      "The current video be processed: subject 008, video 6\n",
      "The current video be processed: subject 008, video 7\n",
      "\n",
      "True Positive: 8, False Posive: 16, False Negative: 25\n",
      "Precision = 0.33333333333333354, Recall =0.24242424242424265, F1-Score = 0.2807017543859654\n",
      "Split 3/29 is processed.\n",
      "\n",
      "Split 4/29 is in process.\n",
      "14 video(s) have been processed.\n",
      "The current video be processed: subject 009, video 3\n",
      "The current video be processed: subject 009, video 4\n",
      "The current video be processed: subject 009, video 6\n",
      "The current video be processed: subject 009, video 7\n",
      "\n",
      "True Positive: 9, False Posive: 20, False Negative: 30\n",
      "Precision = 0.31034482758620713, Recall =0.230769230769231, F1-Score = 0.2647058823529417\n",
      "Split 4/29 is processed.\n",
      "\n",
      "Split 5/29 is in process.\n",
      "18 video(s) have been processed.\n",
      "The current video be processed: subject 010, video 1\n",
      "The current video be processed: subject 010, video 2\n",
      "The current video be processed: subject 010, video 3\n",
      "The current video be processed: subject 010, video 4\n",
      "The current video be processed: subject 010, video 5\n",
      "The current video be processed: subject 010, video 6\n",
      "The current video be processed: subject 010, video 7\n",
      "\n",
      "True Positive: 11, False Posive: 28, False Negative: 47\n",
      "Precision = 0.28205128205128227, Recall =0.18965517241379332, F1-Score = 0.22680412371134065\n",
      "Split 5/29 is processed.\n",
      "\n",
      "Split 6/29 is in process.\n",
      "25 video(s) have been processed.\n",
      "The current video be processed: subject 011, video 1\n",
      "The current video be processed: subject 011, video 2\n",
      "The current video be processed: subject 011, video 3\n",
      "The current video be processed: subject 011, video 4\n",
      "The current video be processed: subject 011, video 5\n",
      "The current video be processed: subject 011, video 6\n",
      "The current video be processed: subject 011, video 7\n",
      "\n",
      "True Positive: 21, False Posive: 48, False Negative: 83\n",
      "Precision = 0.30434782608695676, Recall =0.20192307692307715, F1-Score = 0.24277456647398893\n",
      "Split 6/29 is processed.\n",
      "\n",
      "Split 7/29 is in process.\n",
      "32 video(s) have been processed.\n",
      "The current video be processed: subject 012, video 3\n",
      "The current video be processed: subject 012, video 4\n",
      "The current video be processed: subject 012, video 5\n",
      "The current video be processed: subject 012, video 6\n",
      "The current video be processed: subject 012, video 7\n",
      "\n",
      "True Positive: 22, False Posive: 58, False Negative: 90\n",
      "Precision = 0.27500000000000024, Recall =0.19642857142857165, F1-Score = 0.22916666666666713\n",
      "Split 7/29 is processed.\n",
      "\n",
      "Split 8/29 is in process.\n",
      "37 video(s) have been processed.\n",
      "The current video be processed: subject 013, video 1\n",
      "The current video be processed: subject 013, video 2\n",
      "The current video be processed: subject 013, video 3\n",
      "The current video be processed: subject 013, video 6\n",
      "The current video be processed: subject 013, video 7\n",
      "\n",
      "True Positive: 26, False Posive: 66, False Negative: 100\n",
      "Precision = 0.2826086956521741, Recall =0.20634920634920656, F1-Score = 0.23853211009174355\n",
      "Split 8/29 is processed.\n",
      "\n",
      "Split 9/29 is in process.\n",
      "42 video(s) have been processed.\n",
      "The current video be processed: subject 014, video 1\n",
      "The current video be processed: subject 014, video 2\n",
      "The current video be processed: subject 014, video 3\n",
      "The current video be processed: subject 014, video 4\n",
      "The current video be processed: subject 014, video 5\n",
      "The current video be processed: subject 014, video 6\n",
      "The current video be processed: subject 014, video 7\n",
      "\n",
      "True Positive: 31, False Posive: 80, False Negative: 115\n",
      "Precision = 0.2792792792792795, Recall =0.21232876712328788, F1-Score = 0.24124513618677085\n",
      "Split 9/29 is processed.\n",
      "\n",
      "Split 10/29 is in process.\n",
      "49 video(s) have been processed.\n",
      "The current video be processed: subject 015, video 1\n",
      "The current video be processed: subject 015, video 3\n",
      "The current video be processed: subject 015, video 5\n",
      "The current video be processed: subject 015, video 6\n",
      "The current video be processed: subject 015, video 7\n",
      "\n",
      "True Positive: 32, False Posive: 85, False Negative: 129\n",
      "Precision = 0.27350427350427375, Recall =0.1987577639751555, F1-Score = 0.23021582733812995\n",
      "Split 10/29 is processed.\n",
      "\n",
      "Split 11/29 is in process.\n",
      "54 video(s) have been processed.\n",
      "The current video be processed: subject 016, video 1\n",
      "The current video be processed: subject 016, video 2\n",
      "The current video be processed: subject 016, video 4\n",
      "The current video be processed: subject 016, video 5\n",
      "The current video be processed: subject 016, video 6\n",
      "The current video be processed: subject 016, video 7\n",
      "\n",
      "True Positive: 35, False Posive: 98, False Negative: 137\n",
      "Precision = 0.2631578947368423, Recall =0.20348837209302348, F1-Score = 0.22950819672131192\n",
      "Split 11/29 is processed.\n",
      "\n",
      "Split 12/29 is in process.\n",
      "60 video(s) have been processed.\n",
      "The current video be processed: subject 017, video 1\n",
      "The current video be processed: subject 017, video 2\n",
      "The current video be processed: subject 017, video 3\n",
      "The current video be processed: subject 017, video 4\n",
      "The current video be processed: subject 017, video 5\n",
      "The current video be processed: subject 017, video 6\n",
      "\n",
      "True Positive: 36, False Posive: 106, False Negative: 142\n",
      "Precision = 0.2535211267605636, Recall =0.20224719101123617, F1-Score = 0.22500000000000045\n",
      "Split 12/29 is processed.\n",
      "\n",
      "Split 13/29 is in process.\n",
      "66 video(s) have been processed.\n",
      "The current video be processed: subject 018, video 2\n",
      "The current video be processed: subject 018, video 3\n",
      "The current video be processed: subject 018, video 4\n",
      "The current video be processed: subject 018, video 5\n",
      "The current video be processed: subject 018, video 6\n",
      "The current video be processed: subject 018, video 7\n",
      "\n",
      "True Positive: 41, False Posive: 118, False Negative: 153\n",
      "Precision = 0.257861635220126, Recall =0.21134020618556723, F1-Score = 0.23229461756373984\n",
      "Split 13/29 is processed.\n",
      "\n",
      "Split 14/29 is in process.\n",
      "72 video(s) have been processed.\n",
      "The current video be processed: subject 019, video 1\n",
      "The current video be processed: subject 019, video 2\n",
      "The current video be processed: subject 019, video 3\n",
      "The current video be processed: subject 019, video 5\n",
      "The current video be processed: subject 019, video 7\n",
      "\n",
      "True Positive: 43, False Posive: 126, False Negative: 165\n",
      "Precision = 0.25443786982248545, Recall =0.20673076923076944, F1-Score = 0.22811671087533203\n",
      "Split 14/29 is processed.\n",
      "\n",
      "Split 15/29 is in process.\n",
      "77 video(s) have been processed.\n",
      "The current video be processed: subject 020, video 1\n",
      "The current video be processed: subject 020, video 2\n",
      "The current video be processed: subject 020, video 3\n",
      "The current video be processed: subject 020, video 4\n",
      "The current video be processed: subject 020, video 5\n",
      "The current video be processed: subject 020, video 6\n",
      "The current video be processed: subject 020, video 7\n",
      "\n",
      "True Positive: 43, False Posive: 142, False Negative: 187\n",
      "Precision = 0.23243243243243267, Recall =0.18695652173913066, F1-Score = 0.2072289156626511\n",
      "Split 15/29 is processed.\n",
      "\n",
      "Split 16/29 is in process.\n",
      "84 video(s) have been processed.\n",
      "The current video be processed: subject 021, video 3\n",
      "\n",
      "True Positive: 43, False Posive: 143, False Negative: 189\n",
      "Precision = 0.23118279569892494, Recall =0.18534482758620713, F1-Score = 0.20574162679425884\n",
      "Split 16/29 is processed.\n",
      "\n",
      "Split 17/29 is in process.\n",
      "85 video(s) have been processed.\n",
      "The current video be processed: subject 022, video 2\n",
      "The current video be processed: subject 022, video 3\n",
      "The current video be processed: subject 022, video 6\n",
      "\n",
      "True Positive: 44, False Posive: 152, False Negative: 195\n",
      "Precision = 0.22448979591836757, Recall =0.18410041841004207, F1-Score = 0.20229885057471308\n",
      "Split 17/29 is processed.\n",
      "\n",
      "Split 18/29 is in process.\n",
      "88 video(s) have been processed.\n",
      "The current video be processed: subject 023, video 4\n",
      "\n",
      "True Positive: 45, False Posive: 152, False Negative: 195\n",
      "Precision = 0.2284263959390865, Recall =0.18750000000000022, F1-Score = 0.20594965675057253\n",
      "Split 18/29 is processed.\n",
      "\n",
      "Split 19/29 is in process.\n",
      "89 video(s) have been processed.\n",
      "The current video be processed: subject 024, video 2\n",
      "The current video be processed: subject 024, video 3\n",
      "The current video be processed: subject 024, video 5\n",
      "\n",
      "True Positive: 45, False Posive: 157, False Negative: 202\n",
      "Precision = 0.222772277227723, Recall =0.18218623481781399, F1-Score = 0.20044543429844142\n",
      "Split 19/29 is processed.\n",
      "\n",
      "Split 20/29 is in process.\n",
      "92 video(s) have been processed.\n",
      "The current video be processed: subject 025, video 3\n",
      "The current video be processed: subject 025, video 5\n",
      "\n",
      "True Positive: 45, False Posive: 160, False Negative: 207\n",
      "Precision = 0.21951219512195144, Recall =0.1785714285714288, F1-Score = 0.19693654266958469\n",
      "Split 20/29 is processed.\n",
      "\n",
      "Split 21/29 is in process.\n",
      "94 video(s) have been processed.\n",
      "The current video be processed: subject 026, video 2\n",
      "The current video be processed: subject 026, video 3\n",
      "The current video be processed: subject 026, video 7\n",
      "\n",
      "True Positive: 46, False Posive: 167, False Negative: 214\n",
      "Precision = 0.21596244131455422, Recall =0.17692307692307715, F1-Score = 0.19450317124735775\n",
      "Split 21/29 is processed.\n",
      "\n",
      "Split 22/29 is in process.\n",
      "97 video(s) have been processed.\n",
      "The current video be processed: subject 028, video 4\n",
      "\n",
      "True Positive: 46, False Posive: 168, False Negative: 216\n",
      "Precision = 0.2149532710280376, Recall =0.17557251908396967, F1-Score = 0.19327731092437017\n",
      "Split 22/29 is processed.\n",
      "\n",
      "Split 23/29 is in process.\n",
      "98 video(s) have been processed.\n",
      "The current video be processed: subject 030, video 1\n",
      "The current video be processed: subject 030, video 2\n",
      "The current video be processed: subject 030, video 5\n",
      "\n",
      "True Positive: 47, False Posive: 174, False Negative: 220\n",
      "Precision = 0.21266968325791877, Recall =0.17602996254681672, F1-Score = 0.19262295081967257\n",
      "Split 23/29 is processed.\n",
      "\n",
      "Split 24/29 is in process.\n",
      "101 video(s) have been processed.\n",
      "The current video be processed: subject 032, video 2\n",
      "The current video be processed: subject 032, video 3\n",
      "The current video be processed: subject 032, video 4\n",
      "The current video be processed: subject 032, video 5\n",
      "The current video be processed: subject 032, video 6\n",
      "\n",
      "True Positive: 49, False Posive: 185, False Negative: 228\n",
      "Precision = 0.20940170940170963, Recall =0.176895306859206, F1-Score = 0.19178082191780865\n",
      "Split 24/29 is processed.\n",
      "\n",
      "Split 25/29 is in process.\n",
      "106 video(s) have been processed.\n",
      "The current video be processed: subject 033, video 1\n",
      "The current video be processed: subject 033, video 2\n",
      "The current video be processed: subject 033, video 3\n",
      "The current video be processed: subject 033, video 4\n",
      "The current video be processed: subject 033, video 5\n",
      "The current video be processed: subject 033, video 6\n",
      "The current video be processed: subject 033, video 7\n",
      "\n",
      "True Positive: 50, False Posive: 195, False Negative: 238\n",
      "Precision = 0.20408163265306145, Recall =0.17361111111111133, F1-Score = 0.18761726078799293\n",
      "Split 25/29 is processed.\n",
      "\n",
      "Split 26/29 is in process.\n",
      "113 video(s) have been processed.\n",
      "The current video be processed: subject 034, video 6\n",
      "The current video be processed: subject 034, video 7\n",
      "\n",
      "True Positive: 53, False Posive: 198, False Negative: 239\n",
      "Precision = 0.211155378486056, Recall =0.18150684931506872, F1-Score = 0.19521178637200784\n",
      "Split 26/29 is processed.\n",
      "\n",
      "Split 27/29 is in process.\n",
      "115 video(s) have been processed.\n",
      "The current video be processed: subject 035, video 1\n",
      "The current video be processed: subject 035, video 2\n",
      "The current video be processed: subject 035, video 3\n",
      "The current video be processed: subject 035, video 4\n",
      "The current video be processed: subject 035, video 6\n",
      "The current video be processed: subject 035, video 7\n",
      "\n",
      "True Positive: 55, False Posive: 211, False Negative: 250\n",
      "Precision = 0.2067669172932333, Recall =0.18032786885245924, F1-Score = 0.19264448336252235\n",
      "Split 27/29 is processed.\n",
      "\n",
      "Split 28/29 is in process.\n",
      "121 video(s) have been processed.\n",
      "The current video be processed: subject 036, video 2\n",
      "The current video be processed: subject 036, video 4\n",
      "The current video be processed: subject 036, video 6\n",
      "The current video be processed: subject 036, video 7\n",
      "\n",
      "True Positive: 60, False Posive: 217, False Negative: 262\n",
      "Precision = 0.21660649819494607, Recall =0.1863354037267083, F1-Score = 0.20033388981636105\n",
      "Split 28/29 is processed.\n",
      "\n",
      "Split 29/29 is in process.\n",
      "125 video(s) have been processed.\n",
      "The current video be processed: subject 037, video 2\n",
      "The current video be processed: subject 037, video 3\n",
      "The current video be processed: subject 037, video 4\n",
      "The current video be processed: subject 037, video 5\n",
      "The current video be processed: subject 037, video 7\n",
      "\n",
      "True Positive: 60, False Posive: 226, False Negative: 271\n",
      "Precision = 0.20979020979021, Recall =0.1812688821752268, F1-Score = 0.19448946515397125\n",
      "Split 29/29 is processed.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metric_fn, result_dict = functions.spot_and_evaluate(\n",
    "    preds,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    clean_videos_images,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_code,\n",
    "    k,\n",
    "    p=0.60,\n",
    "    show_plot_or_not=False,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True Positive: 60, False Posive: 226, False Negative: 271\n",
      "Final Precision = 0.20979020979021,\n",
      "Final Recall =0.1812688821752268,\n",
      "Final F1-Score = 0.19448946515397125\n",
      "\n",
      "Highest Precision = 0.33333333333333354,\n",
      "Highest Recall =0.24242424242424265,\n",
      "Highest F1-Score = 0.2807017543859654\n"
     ]
    }
   ],
   "source": [
    "functions.final_evaluate(metric_fn, result_dict)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value |\n",
    "| --- | --- | --- | --- |\n",
    "| model | 3D-CNN | SL-Swin-T | SL-Swin-T |\n",
    "| epochs | | 25 (9264 m) | 25 (9264 m) |\n",
    "| batch_size | | 48 | 48 |\n",
    "| p | | 0.59 | 0.60 |\n",
    "| True Positive | | 54 | 52 |\n",
    "| False Positive | | 240 | 238 |\n",
    "| False Negative | | 277 | 279 |\n",
    "| Precision | | 0.1836 | 0.1793 |\n",
    "| Recall | | 0.1631 | 0.1571 |\n",
    "| F1-Score | 0.1595 | 0.1728 | 0.1675 |\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value |\n",
    "| --- | --- | --- |\n",
    "| model | S-Swin-T | L-Swin-T |\n",
    "| epochs | 25 ( m) | 25 (14027 m) |\n",
    "| batch_size | 48 | 48 |\n",
    "| p | 0.59 | 0.60 |\n",
    "| True Positive | | |\n",
    "| False Positive | | |\n",
    "| False Negative | | |\n",
    "| Precision | 0. | 0. |\n",
    "| Recall | 0. | 0. |\n",
    "| F1-Score | 0. | 0. |\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ablation Study\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " p | TP | FP | FN | Precision | Recall | F1-Score\n",
      "0.01 | 118 | 810 | 213 | 0.1272 | 0.3565 | 0.1875 |\n",
      "0.02 | 116 | 794 | 215 | 0.1275 | 0.3505 | 0.1869 |\n",
      "0.03 | 115 | 775 | 216 | 0.1292 | 0.3474 | 0.1884 |\n",
      "0.04 | 113 | 764 | 218 | 0.1288 | 0.3414 | 0.1871 |\n",
      "0.05 | 113 | 749 | 218 | 0.1311 | 0.3414 | 0.1894 |\n",
      "0.06 | 111 | 737 | 220 | 0.1309 | 0.3353 | 0.1883 |\n",
      "0.07 | 110 | 719 | 221 | 0.1327 | 0.3323 | 0.1897 |\n",
      "0.08 | 108 | 704 | 223 | 0.1330 | 0.3263 | 0.1890 |\n",
      "0.09 | 108 | 693 | 223 | 0.1348 | 0.3263 | 0.1908 |\n",
      "0.10 | 107 | 676 | 224 | 0.1367 | 0.3233 | 0.1921 |\n",
      "0.11 | 105 | 662 | 226 | 0.1369 | 0.3172 | 0.1913 |\n",
      "0.12 | 103 | 649 | 228 | 0.1370 | 0.3112 | 0.1902 |\n",
      "0.13 | 101 | 634 | 230 | 0.1374 | 0.3051 | 0.1895 |\n",
      "0.14 | 100 | 621 | 231 | 0.1387 | 0.3021 | 0.1901 |\n",
      "0.15 | 99 | 607 | 232 | 0.1402 | 0.2991 | 0.1909 |\n",
      "0.16 | 99 | 599 | 232 | 0.1418 | 0.2991 | 0.1924 |\n",
      "0.17 | 98 | 584 | 233 | 0.1437 | 0.2961 | 0.1935 |\n",
      "0.18 | 97 | 565 | 234 | 0.1465 | 0.2931 | 0.1954 |\n",
      "0.19 | 97 | 553 | 234 | 0.1492 | 0.2931 | 0.1978 |\n",
      "0.20 | 97 | 536 | 234 | 0.1532 | 0.2931 | 0.2012 |\n",
      "0.21 | 97 | 524 | 234 | 0.1562 | 0.2931 | 0.2038 |\n",
      "0.22 | 94 | 515 | 237 | 0.1544 | 0.2840 | 0.2000 |\n",
      "0.23 | 93 | 505 | 238 | 0.1555 | 0.2810 | 0.2002 |\n",
      "0.24 | 93 | 494 | 238 | 0.1584 | 0.2810 | 0.2026 |\n",
      "0.25 | 92 | 482 | 239 | 0.1603 | 0.2779 | 0.2033 |\n",
      "0.26 | 91 | 475 | 240 | 0.1608 | 0.2749 | 0.2029 |\n",
      "0.27 | 89 | 465 | 242 | 0.1606 | 0.2689 | 0.2011 |\n",
      "0.28 | 87 | 456 | 244 | 0.1602 | 0.2628 | 0.1991 |\n",
      "0.29 | 85 | 448 | 246 | 0.1595 | 0.2568 | 0.1968 |\n",
      "0.30 | 83 | 435 | 248 | 0.1602 | 0.2508 | 0.1955 |\n",
      "0.31 | 82 | 427 | 249 | 0.1611 | 0.2477 | 0.1952 |\n",
      "0.32 | 78 | 417 | 253 | 0.1576 | 0.2356 | 0.1889 |\n",
      "0.33 | 77 | 406 | 254 | 0.1594 | 0.2326 | 0.1892 |\n",
      "0.34 | 77 | 400 | 254 | 0.1614 | 0.2326 | 0.1906 |\n",
      "0.35 | 77 | 395 | 254 | 0.1631 | 0.2326 | 0.1918 |\n",
      "0.36 | 75 | 387 | 256 | 0.1623 | 0.2266 | 0.1892 |\n",
      "0.37 | 74 | 376 | 257 | 0.1644 | 0.2236 | 0.1895 |\n",
      "0.38 | 74 | 367 | 257 | 0.1678 | 0.2236 | 0.1917 |\n",
      "0.39 | 72 | 357 | 259 | 0.1678 | 0.2175 | 0.1895 |\n",
      "0.40 | 72 | 351 | 259 | 0.1702 | 0.2175 | 0.1910 |\n",
      "0.41 | 71 | 346 | 260 | 0.1703 | 0.2145 | 0.1898 |\n",
      "0.42 | 70 | 342 | 261 | 0.1699 | 0.2115 | 0.1884 |\n",
      "0.43 | 67 | 337 | 264 | 0.1658 | 0.2024 | 0.1823 |\n",
      "0.44 | 67 | 326 | 264 | 0.1705 | 0.2024 | 0.1851 |\n",
      "0.45 | 67 | 319 | 264 | 0.1736 | 0.2024 | 0.1869 |\n",
      "0.46 | 67 | 314 | 264 | 0.1759 | 0.2024 | 0.1882 |\n",
      "0.47 | 67 | 307 | 264 | 0.1791 | 0.2024 | 0.1901 |\n",
      "0.48 | 67 | 301 | 264 | 0.1821 | 0.2024 | 0.1917 |\n",
      "0.49 | 67 | 292 | 264 | 0.1866 | 0.2024 | 0.1942 |\n",
      "0.50 | 65 | 287 | 266 | 0.1847 | 0.1964 | 0.1903 |\n",
      "0.51 | 63 | 278 | 268 | 0.1848 | 0.1903 | 0.1875 |\n",
      "0.52 | 63 | 271 | 268 | 0.1886 | 0.1903 | 0.1895 |\n",
      "0.53 | 63 | 268 | 268 | 0.1903 | 0.1903 | 0.1903 |\n",
      "0.54 | 63 | 257 | 268 | 0.1969 | 0.1903 | 0.1935 |\n",
      "0.55 | 63 | 248 | 268 | 0.2026 | 0.1903 | 0.1963 |\n",
      "0.56 | 62 | 245 | 269 | 0.2020 | 0.1873 | 0.1944 |\n",
      "0.57 | 61 | 241 | 270 | 0.2020 | 0.1843 | 0.1927 |\n",
      "0.58 | 61 | 235 | 270 | 0.2061 | 0.1843 | 0.1946 |\n",
      "0.59 | 61 | 231 | 270 | 0.2089 | 0.1843 | 0.1958 |\n",
      "0.60 | 60 | 226 | 271 | 0.2098 | 0.1813 | 0.1945 |\n",
      "0.61 | 60 | 223 | 271 | 0.2120 | 0.1813 | 0.1954 |\n",
      "0.62 | 58 | 220 | 273 | 0.2086 | 0.1752 | 0.1905 |\n",
      "0.63 | 56 | 217 | 275 | 0.2051 | 0.1692 | 0.1854 |\n",
      "0.64 | 56 | 216 | 275 | 0.2059 | 0.1692 | 0.1857 |\n",
      "0.65 | 56 | 208 | 275 | 0.2121 | 0.1692 | 0.1882 |\n",
      "0.66 | 56 | 204 | 275 | 0.2154 | 0.1692 | 0.1895 |\n",
      "0.67 | 56 | 200 | 275 | 0.2188 | 0.1692 | 0.1908 |\n",
      "0.68 | 54 | 198 | 277 | 0.2143 | 0.1631 | 0.1852 |\n",
      "0.69 | 52 | 195 | 279 | 0.2105 | 0.1571 | 0.1799 |\n",
      "0.70 | 51 | 189 | 280 | 0.2125 | 0.1541 | 0.1786 |\n",
      "0.71 | 49 | 187 | 282 | 0.2076 | 0.1480 | 0.1728 |\n",
      "0.72 | 48 | 184 | 283 | 0.2069 | 0.1450 | 0.1705 |\n",
      "0.73 | 46 | 181 | 285 | 0.2026 | 0.1390 | 0.1649 |\n",
      "0.74 | 46 | 177 | 285 | 0.2063 | 0.1390 | 0.1661 |\n",
      "0.75 | 45 | 167 | 286 | 0.2123 | 0.1360 | 0.1657 |\n",
      "0.76 | 45 | 164 | 286 | 0.2153 | 0.1360 | 0.1667 |\n",
      "0.77 | 43 | 161 | 288 | 0.2108 | 0.1299 | 0.1607 |\n",
      "0.78 | 42 | 159 | 289 | 0.2090 | 0.1269 | 0.1579 |\n",
      "0.79 | 41 | 153 | 290 | 0.2113 | 0.1239 | 0.1562 |\n",
      "0.80 | 40 | 149 | 291 | 0.2116 | 0.1208 | 0.1538 |\n",
      "0.81 | 39 | 142 | 292 | 0.2155 | 0.1178 | 0.1523 |\n",
      "0.82 | 39 | 140 | 292 | 0.2179 | 0.1178 | 0.1529 |\n",
      "0.83 | 39 | 139 | 292 | 0.2191 | 0.1178 | 0.1532 |\n",
      "0.84 | 39 | 134 | 292 | 0.2254 | 0.1178 | 0.1548 |\n",
      "0.85 | 37 | 131 | 294 | 0.2202 | 0.1118 | 0.1483 |\n",
      "0.86 | 37 | 127 | 294 | 0.2256 | 0.1118 | 0.1495 |\n",
      "0.87 | 37 | 123 | 294 | 0.2313 | 0.1118 | 0.1507 |\n",
      "0.88 | 35 | 121 | 296 | 0.2244 | 0.1057 | 0.1437 |\n",
      "0.89 | 35 | 119 | 296 | 0.2273 | 0.1057 | 0.1443 |\n",
      "0.90 | 35 | 117 | 296 | 0.2303 | 0.1057 | 0.1449 |\n",
      "0.91 | 35 | 114 | 296 | 0.2349 | 0.1057 | 0.1458 |\n",
      "0.92 | 35 | 113 | 296 | 0.2365 | 0.1057 | 0.1461 |\n",
      "0.93 | 35 | 113 | 296 | 0.2365 | 0.1057 | 0.1461 |\n",
      "0.94 | 33 | 110 | 298 | 0.2308 | 0.0997 | 0.1392 |\n",
      "0.95 | 31 | 106 | 300 | 0.2263 | 0.0937 | 0.1325 |\n",
      "0.96 | 30 | 104 | 301 | 0.2239 | 0.0906 | 0.1290 |\n",
      "0.97 | 28 | 104 | 303 | 0.2121 | 0.0846 | 0.1210 |\n",
      "0.98 | 28 | 104 | 303 | 0.2121 | 0.0846 | 0.1210 |\n",
      "0.99 | 28 | 103 | 303 | 0.2137 | 0.0846 | 0.1212 |\n"
     ]
    }
   ],
   "source": [
    "ablation_dict = functions.ablation_study_p_dev(\n",
    "    preds,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    clean_videos_images,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_code,\n",
    "    k,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6696e3028ae55fa5be357812587f73779dfb157cc851dab91c4ca8ffd3c7a806"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}