test_word_encoding.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This shows that the string encoding work correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from typing import Dict, List, Optional, Tuple\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Precompile regular expressions\n",
    "# Based on https://github.com/yoonkim/CNN_sentence/blob/23e0e1f7355705bb083043fda05c031b15acb38c/process_data.py#L97\n",
    "RE_NOT_ALNUM = regex.compile(r\"[^A-Za-z0-9(),!?'`]\")\n",
    "RE_CONTRACTION = regex.compile(r\"'([std]|ve|re|ll)\")\n",
    "RE_PUNCTUATION = regex.compile(r\"([(),!?'`])\")\n",
    "RE_WHITESPACE = regex.compile(r\"\\s{2,}\")\n",
    "\n",
    "def clean_str(text):\n",
    "    text = RE_NOT_ALNUM.sub(\" \", text)\n",
    "    text = RE_CONTRACTION.sub(\" '\\\\1\", text)\n",
    "    text = RE_PUNCTUATION.sub(\" \\\\1 \", text)\n",
    "    # TODO: remove stop words here?\n",
    "    text = RE_WHITESPACE.sub(\" \", text)\n",
    "    return text.strip().lower().split()\n",
    "\n",
    "def generate_word_freq(*series: pd.Series, max_n: Optional[int] = None) -> List[Tuple[str, int]]:\n",
    "    word_counts = Counter()\n",
    "    for s in series:\n",
    "        for row in s:\n",
    "            tokens = clean_str(row)\n",
    "            word_counts.update(tokens)\n",
    "    return word_counts.most_common(max_n)\n",
    "\n",
    "def generate_word_to_idx(*series: pd.Series, max_n: Optional[int] = None) -> Dict[str, int]:\n",
    "    word_freq = generate_word_freq(*series, max_n=max_n)\n",
    "    return {word: idx for idx, (word, _) in enumerate(word_freq)}\n",
    "\n",
    "def bag_of_words(sentences: pd.Series, word_to_idx: Dict[str, int]):\n",
    "    sentences = sentences.apply(clean_str)\n",
    "    # lil_matrix not supported, is flattened in TMU\n",
    "    # Using np.uint32 as it's the same in TMU\n",
    "    matrix = np.zeros((len(sentences), len(word_to_idx)), dtype=np.uint32)\n",
    "    for i, l in enumerate(sentences):\n",
    "        for w in l:\n",
    "            if w in word_to_idx:\n",
    "                matrix[i, word_to_idx[w]] = 1\n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>This is the first sentence.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The second sentence is here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sentence number three is long.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The fourth sentence is short.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>This is the last sentence in the set.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Another set of sentences begins here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>This is the second sentence in the new set.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>The third sentence is the longest one yet.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>A fourth sentence is here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>The last sentence a a a in this set is quite s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0                        This is the first sentence.\n",
       "1                       The second sentence is here.\n",
       "2                     Sentence number three is long.\n",
       "3                      The fourth sentence is short.\n",
       "4              This is the last sentence in the set.\n",
       "5              Another set of sentences begins here.\n",
       "6        This is the second sentence in the new set.\n",
       "7         The third sentence is the longest one yet.\n",
       "8                         A fourth sentence is here.\n",
       "9  The last sentence a a a in this set is quite s..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"text\": [\n",
    "    \"This is the first sentence.\",\n",
    "    \"The second sentence is here.\",\n",
    "    \"Sentence number three is long.\",\n",
    "    \"The fourth sentence is short.\",\n",
    "    \"This is the last sentence in the set.\",\n",
    "    \"Another set of sentences begins here.\",\n",
    "    \"This is the second sentence in the new set.\",\n",
    "    \"The third sentence is the longest one yet.\",\n",
    "    \"A fourth sentence is here.\",\n",
    "    \"The last sentence a a a in this set is quite short.\"\n",
    "]})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'the': 0,\n",
       " 'is': 1,\n",
       " 'sentence': 2,\n",
       " 'this': 3,\n",
       " 'set': 4,\n",
       " 'a': 5,\n",
       " 'here': 6,\n",
       " 'in': 7,\n",
       " 'second': 8,\n",
       " 'fourth': 9}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_to_idx = generate_word_to_idx(df[\"text\"], max_n=10)\n",
    "word_to_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, 1, 1, 0, 0, 0, 0, 0, 0],\n",
       "       [1, 1, 1, 0, 0, 0, 1, 0, 1, 0],\n",
       "       [0, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [1, 1, 1, 0, 0, 0, 0, 0, 0, 1],\n",
       "       [1, 1, 1, 1, 1, 0, 0, 1, 0, 0],\n",
       "       [0, 0, 0, 0, 1, 0, 1, 0, 0, 0],\n",
       "       [1, 1, 1, 1, 1, 0, 0, 1, 1, 0],\n",
       "       [1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [0, 1, 1, 0, 0, 1, 1, 0, 0, 1],\n",
       "       [1, 1, 1, 1, 1, 1, 0, 1, 0, 0]], dtype=uint32)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix = bag_of_words(df[\"text\"], word_to_idx)\n",
    "matrix"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "thesis-g87dtkxN-py3.8",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}