translation.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        },
        "id": "D0N250FJeukD",
        "outputId": "79b754c1-9f11-4e8f-8aff-e887074030c3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>native_text</th>\n",
              "      <th>translation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>indulo anno assami masaladinusulu kuda vesi ch...</td>\n",
              "      <td>ఇందులో అన్నో అస్సామి మసలాదినుసులు కుడ వేసి చేస...</td>\n",
              "      <td>A lot of Assamese spices also go into its making.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ondu all, one tadupusin erandu doskayp botukko...</td>\n",
              "      <td>ஒண்டு ஆல், ஒனே தடுப்பூசின் இரண்டு டாஸ்கேப் போட...</td>\n",
              "      <td>Not one, but two doses of the same vaccine.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>tumhen mal main jane yaa train yaa flite main ...</td>\n",
              "      <td>तुम्हें मल मैन जाने या ट्रेन या फ्लाइट मैन ट्र...</td>\n",
              "      <td>You will be asked to show your vaccine certifi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  \\\n",
              "0  indulo anno assami masaladinusulu kuda vesi ch...   \n",
              "1  ondu all, one tadupusin erandu doskayp botukko...   \n",
              "2  tumhen mal main jane yaa train yaa flite main ...   \n",
              "\n",
              "                                         native_text  \\\n",
              "0  ఇందులో అన్నో అస్సామి మసలాదినుసులు కుడ వేసి చేస...   \n",
              "1  ஒண்டு ஆல், ஒனே தடுப்பூசின் இரண்டு டாஸ்கேப் போட...   \n",
              "2  तुम्हें मल मैन जाने या ट्रेन या फ्लाइट मैन ट्र...   \n",
              "\n",
              "                                         translation  \n",
              "0  A lot of Assamese spices also go into its making.  \n",
              "1        Not one, but two doses of the same vaccine.  \n",
              "2  You will be asked to show your vaccine certifi...  "
            ]
          },
          "execution_count": 1,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv('task2.csv')\n",
        "df.head(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "4nJ8nQw3faJy"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train, test = train_test_split(df, test_size=0.1, random_state=0)\n",
        "\n",
        "train.reset_index(inplace=True)\n",
        "test.reset_index(inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_yYcL401m5LR",
        "outputId": "1b53e731-41f2-46ad-820b-d69efe9fcbcd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "850\n"
          ]
        }
      ],
      "source": [
        "print(len(test))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GiL48i6PfL0L"
      },
      "outputs": [],
      "source": [
        "from transformers import NllbTokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ALGuyW6ZfsLy",
        "outputId": "f7c91a88-aad2-45c6-d04c-670864ac2f82"
      },
      "outputs": [],
      "source": [
        "tokenizer = NllbTokenizer.from_pretrained('facebook/nllb-200-distilled-600M')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "yyRaywk1iryb"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "\n",
        "class TranslationDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, data, tokenizer, max_len):\n",
        "        super(TranslationDataset, self).__init__()\n",
        "\n",
        "        self.data=data['native_text']\n",
        "        self.target = data['translation']\n",
        "        self.tokenizer = tokenizer\n",
        "        self.max_len = max_len\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        text = self.data[idx]\n",
        "        translation = self.target[idx]\n",
        "\n",
        "        source = self.tokenizer.encode_plus(\n",
        "            text,\n",
        "            return_tensors='pt',\n",
        "            truncation=True,\n",
        "            max_length=self.max_len,\n",
        "            padding='max_length'\n",
        "        )\n",
        "        target = self.tokenizer.encode_plus(\n",
        "            translation,\n",
        "            return_tensors='pt',\n",
        "            truncation=True,\n",
        "            max_length=self.max_len,\n",
        "            padding='max_length'\n",
        "        )\n",
        "\n",
        "        return {\n",
        "            'input_ids': source['input_ids'].flatten(),\n",
        "            'attention_mask': source['attention_mask'].flatten(),\n",
        "            'labels': target['input_ids'].flatten(),\n",
        "        }\n",
        "\n",
        "train_ds = TranslationDataset(train, tokenizer, 512)\n",
        "test_ds = TranslationDataset(test, tokenizer, 512)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "StExqZG-gWD6"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModelForSeq2SeqLM\n",
        "\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained('facebook/nllb-200-distilled-600M')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "WXfMxv3sgiLn"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "M2M100ForConditionalGeneration(\n",
              "  (model): M2M100Model(\n",
              "    (shared): Embedding(256206, 1024, padding_idx=1)\n",
              "    (encoder): M2M100Encoder(\n",
              "      (embed_tokens): Embedding(256206, 1024, padding_idx=1)\n",
              "      (embed_positions): M2M100SinusoidalPositionalEmbedding()\n",
              "      (layers): ModuleList(\n",
              "        (0-11): 12 x M2M100EncoderLayer(\n",
              "          (self_attn): M2M100Attention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (activation_fn): ReLU()\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "      (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "    (decoder): M2M100Decoder(\n",
              "      (embed_tokens): Embedding(256206, 1024, padding_idx=1)\n",
              "      (embed_positions): M2M100SinusoidalPositionalEmbedding()\n",
              "      (layers): ModuleList(\n",
              "        (0-11): 12 x M2M100DecoderLayer(\n",
              "          (self_attn): M2M100Attention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (activation_fn): ReLU()\n",
              "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (encoder_attn): M2M100Attention(\n",
              "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "          )\n",
              "          (encoder_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        )\n",
              "      )\n",
              "      (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "  )\n",
              "  (lm_head): Linear(in_features=1024, out_features=256206, bias=False)\n",
              ")"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.cuda()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "aAG9tLv_gqHa"
      },
      "outputs": [],
      "source": [
        "from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments\n",
        "\n",
        "# Define training arguments for the model\n",
        "training_args = Seq2SeqTrainingArguments(\n",
        "    output_dir='./results',\n",
        "    num_train_epochs=5,\n",
        "    per_device_train_batch_size=8,\n",
        "    per_device_eval_batch_size=8,\n",
        "    warmup_steps=1000,\n",
        "    weight_decay=0.01,\n",
        "    logging_dir='./logs',\n",
        "    logging_steps=10,\n",
        "    evaluation_strategy=\"epoch\",\n",
        "    report_to='none'\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nw01B4icZNyu"
      },
      "outputs": [],
      "source": [
        "# Initializing the Trainer object\n",
        "trainer = Seq2SeqTrainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_ds,\n",
        "    eval_dataset=test_ds,\n",
        ")\n",
        "\n",
        "# Starting the training process\n",
        "trainer.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "ED197ycHg5HL"
      },
      "outputs": [],
      "source": [
        "def translate(text):\n",
        "    inputs = tokenizer(\n",
        "        text,\n",
        "        return_tensors='pt',\n",
        "        padding='max_length',\n",
        "        truncation=True,\n",
        "        max_length=512\n",
        "    )\n",
        "    result = model.generate(\n",
        "        **inputs.to(model.device),\n",
        "        num_beams=4,\n",
        "        forced_bos_token_id=tokenizer.lang_code_to_id[\"eng_Latn\"]\n",
        "    )\n",
        "    tr = tokenizer.batch_decode(result, skip_special_tokens=True)\n",
        "    return tr[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "out = pd.read_csv('translation.csv')\n",
        "\n",
        "out['output'] = out['native_text'].apply(lambda x: translate(x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "AHfsxbUynQpa",
        "outputId": "a9b0002a-4558-4f31-c809-df77445dd395"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>text</th>\n",
              "      <th>native_text</th>\n",
              "      <th>translation</th>\n",
              "      <th>output</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>8101</td>\n",
              "      <td>arsiaiku,hyderabadloni udyogul state insurence...</td>\n",
              "      <td>ఆర్సియైకు,హైదరాబాద్లోని ఉద్యోగుల్ స్టేట్ ఇన్షు...</td>\n",
              "      <td>Employees' State Insurance Corporation Medical...</td>\n",
              "      <td>The RCI, Employees' State Insurance Corporatio...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>5597</td>\n",
              "      <td>olive bachchi sifan putwai udutirunth batma pa...</td>\n",
              "      <td>ஒலிவ் பச்சி சிஃபான் புட்வை உடுதிருந்த் பாத்மா ...</td>\n",
              "      <td>Clad in an olive-green chiffon saree, Fatma lo...</td>\n",
              "      <td>I felt like I was talking about an olive grove...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>394</td>\n",
              "      <td>ih bohut vadhia hai|</td>\n",
              "      <td>ਇਹ ਬੋਹੁਤ ਵਧਿਆ ਹਾਈ|</td>\n",
              "      <td>That's great.</td>\n",
              "      <td>It's very hectic.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index                                               text  \\\n",
              "0   8101  arsiaiku,hyderabadloni udyogul state insurence...   \n",
              "1   5597  olive bachchi sifan putwai udutirunth batma pa...   \n",
              "2    394                               ih bohut vadhia hai|   \n",
              "\n",
              "                                         native_text  \\\n",
              "0  ఆర్సియైకు,హైదరాబాద్లోని ఉద్యోగుల్ స్టేట్ ఇన్షు...   \n",
              "1  ஒலிவ் பச்சி சிஃபான் புட்வை உடுதிருந்த் பாத்மா ...   \n",
              "2                                 ਇਹ ਬੋਹੁਤ ਵਧਿਆ ਹਾਈ|   \n",
              "\n",
              "                                         translation  \\\n",
              "0  Employees' State Insurance Corporation Medical...   \n",
              "1  Clad in an olive-green chiffon saree, Fatma lo...   \n",
              "2                                      That's great.   \n",
              "\n",
              "                                              output  \n",
              "0  The RCI, Employees' State Insurance Corporatio...  \n",
              "1  I felt like I was talking about an olive grove...  \n",
              "2                                  It's very hectic.  "
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "out.head(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [],
      "source": [
        "out.to_csv('task2_output.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9lQj3FkRtTH_",
        "outputId": "231f69d2-8f25-480e-b4ec-d04dce6be446"
      },
      "outputs": [],
      "source": [
        "%pip install -q sacrebleu\n",
        "%pip install -q rouge_score\n",
        "%pip install -q datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oKIGWEv3hWbY"
      },
      "outputs": [],
      "source": [
        "import sacrebleu\n",
        "from datasets import load_metric\n",
        "\n",
        "rouge = load_metric('rouge')\n",
        "bleu_calc = sacrebleu.BLEU()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "test['output'] = test['native_text'].apply(lambda x: translate(x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "djY-qeT6hZUg",
        "outputId": "87ae497c-61fc-42eb-d8f2-895b734e3590"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "BLEU = 35.09 62.6/39.7/28.6/21.5 (BP = 0.998 ratio = 0.998 hyp_len = 31568 ref_len = 31629)\n"
          ]
        }
      ],
      "source": [
        "print(bleu_calc.corpus_score(test['output'].tolist(), [test['translation'].tolist()]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "3T_n49SjbPuH"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'rouge1': AggregateScore(low=Score(precision=0.5982260015044203, recall=0.586687728720013, fmeasure=0.588395071327604), mid=Score(precision=0.6090383968353283, recall=0.5979467457017437, fmeasure=0.5991336826345741), high=Score(precision=0.6205648086911699, recall=0.6101997731202324, fmeasure=0.610703677963987)), 'rouge2': AggregateScore(low=Score(precision=0.38572363995227416, recall=0.37979733211233024, fmeasure=0.380110453125751), mid=Score(precision=0.3993735371702426, recall=0.3930406813805263, fmeasure=0.3934213602422587), high=Score(precision=0.4125526452542492, recall=0.4062831283614229, fmeasure=0.4067131804773679)), 'rougeL': AggregateScore(low=Score(precision=0.5582783074140499, recall=0.5484048818106864, fmeasure=0.5494473633445973), mid=Score(precision=0.5699244345604186, recall=0.5603384416674695, fmeasure=0.5612858677556972), high=Score(precision=0.5819841724745507, recall=0.5723916645499619, fmeasure=0.573080584554977)), 'rougeLsum': AggregateScore(low=Score(precision=0.5584974698078362, recall=0.5491859786564778, fmeasure=0.5499980212426068), mid=Score(precision=0.5698907375445904, recall=0.5603766322361721, fmeasure=0.5611989530465493), high=Score(precision=0.5811275193787573, recall=0.5722195387226563, fmeasure=0.5724474271719822))}\n"
          ]
        }
      ],
      "source": [
        "rouge_score = rouge.compute(predictions=test['output'].tolist(), references=test['translation'].tolist())\n",
        "print(rouge_score)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.18"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}