01_Twitter Scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "mpl.rcParams['figure.facecolor'] = 'white'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from requests.structures import CaseInsensitiveDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import dotenv_values\n",
    "\n",
    "config = dotenv_values(\".env\")  # config = {\"USER\": \"foo\", \"EMAIL\": \"foo@example.org\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [],
   "source": [
    "BEARER_TOKEN = config.BEARER_TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "url1 = \"https://api.twitter.com/2/users/by/username/LulaOficial\" \n",
    "nome_candidato = 'LulaOficial' # 2670726740\n",
    "\n",
    "# url1 = \"https://api.twitter.com/2/users/by/username/jairbolsonaro\" # 128372940\n",
    "# nome_candidato = 'jairbolsonaro'\n",
    "\n",
    "headers = CaseInsensitiveDict()\n",
    "headers[\"authorization\"] = f\"Bearer {BEARER_TOKEN}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get userid based on username\n",
    "response = requests.get(url1, headers=headers)\n",
    "data = response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2670726740'"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_id = data['data']['id']\n",
    "user_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters\n",
    "start_time = '2022-01-01T00:00:00.00-03:00'\n",
    "end_time = '2022-06-30T00:00:00.00-03:00'\n",
    "\n",
    "# start_time = '2022-01-01T00:00:00.00-03:00'\n",
    "# end_time = '2022-01-02T00:00:00.00-03:00'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = f\"https://api.twitter.com/2/users/{user_id}/tweets?\" \\\n",
    "    f\"expansions=entities.mentions.username,author_id,referenced_tweets.id,attachments.media_keys&\" \\\n",
    "    f\"tweet.fields=created_at,author_id,lang,public_metrics,referenced_tweets,attachments&\" \\\n",
    "    f\"media.fields=preview_image_url,url,type&\" \\\n",
    "    f\"start_time={start_time}&end_time={end_time}&max_results=100\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_media_referenced(media_referenced_tweets_id):\n",
    "\n",
    "    if media_referenced_tweets_id == \"\":\n",
    "        return\n",
    "\n",
    "    url_single = f\"https://api.twitter.com/2/tweets/{media_referenced_tweets_id}?\" \\\n",
    "        f\"tweet.fields=lang&expansions=attachments.media_keys&media.fields=preview_image_url,url\"\n",
    "        \n",
    "    response = requests.get(url_single, headers=headers)\n",
    "    data = response.json()\n",
    "    \n",
    "    if 'includes' in data.keys():\n",
    "        data = data['includes']\n",
    "        if 'media' in data.keys():\n",
    "            return data['media'][0]['type']\n",
    "\n",
    "    return \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_dic = {\"id\": [], \"created_at\": [], \"text\": [], \n",
    "\"retweet_count\": [], \"reply_count\": [], \"like_count\": [], \"quote_count\": [], \n",
    "\"lang\": [], \"referenced_tweets_type\": [], \"referenced_tweets_id\": [], 'referenced_tweets_qty': [], 'referenced_tweets_author': [],\n",
    "'media_key': [], 'media_qty': [], 'media_type': [],\n",
    "'mentions': [], 'qtd_mencoes': []}\n",
    "\n",
    "next_token = \"\"\n",
    "\n",
    "while True:\n",
    "\n",
    "    url = f\"{base_url}&pagination_token={next_token}\" if next_token else base_url\n",
    "    \n",
    "    response = requests.get(url, headers=headers)\n",
    "    data = response.json()\n",
    "\n",
    "    tweets = data['data']\n",
    "\n",
    "    for tweet in tweets:\n",
    "        tweets_dic['id'].append(tweet['id'])\n",
    "        tweets_dic['created_at'].append(tweet['created_at'])\n",
    "        tweets_dic['text'].append(tweet['text'])\n",
    "        tweets_dic['retweet_count'].append(tweet['public_metrics']['retweet_count'])\n",
    "        tweets_dic['reply_count'].append(tweet['public_metrics']['reply_count'])\n",
    "        tweets_dic['like_count'].append(tweet['public_metrics']['like_count'])\n",
    "        tweets_dic['quote_count'].append(tweet['public_metrics']['quote_count'])\n",
    "\n",
    "        tweets_dic['lang'].append(tweet.get('lang', ''))\n",
    "\n",
    "\n",
    "        referenced_tweets_id = '' \n",
    "        author_referenced_tweet = ''\n",
    "        if 'referenced_tweets' in tweet.keys():\n",
    "            tweets_dic['referenced_tweets_type'].append(tweet['referenced_tweets'][0]['type'])\n",
    "            referenced_tweets_id = tweet['referenced_tweets'][0]['id']\n",
    "            tweets_dic['referenced_tweets_id'].append(referenced_tweets_id)\n",
    "            tweets_dic['referenced_tweets_qty'].append(len(tweet['referenced_tweets']))\n",
    "\n",
    "            if tweet['referenced_tweets'][0]['type'] == 'retweeted':\n",
    "                author_referenced_tweet = tweet['text'][4:].split(':')[0]\n",
    "\n",
    "            else:\n",
    "                author_referenced_tweet = [ tweet_includes['author_id'] for tweet_includes in data['includes']['tweets'] if tweet_includes['id'] == tweet['referenced_tweets'][0]['id']]\n",
    "                if not author_referenced_tweet:\n",
    "                    author_referenced_tweet = 0\n",
    "                else:\n",
    "                    username =  [ user_includes['username'] for user_includes in data['includes']['users'] if user_includes['id'] == author_referenced_tweet[0] ]\n",
    "                    if len(username):\n",
    "                        author_referenced_tweet = username[0]\n",
    "                    else:\n",
    "                        author_referenced_tweet = author_referenced_tweet[0]\n",
    "\n",
    "            tweets_dic['referenced_tweets_author'].append(author_referenced_tweet)\n",
    "\n",
    "        else:\n",
    "            tweets_dic['referenced_tweets_type'].append('')\n",
    "            tweets_dic['referenced_tweets_id'].append('')\n",
    "            tweets_dic['referenced_tweets_qty'].append(0)\n",
    "\n",
    "            tweets_dic['referenced_tweets_author'].append('')\n",
    "\n",
    "\n",
    "        if 'attachments' in tweet.keys():\n",
    "            media_key = tweet['attachments']['media_keys'][0]\n",
    "            tweets_dic['media_key'].append(media_key)\n",
    "            tweets_dic['media_qty'].append(len(tweet['attachments']['media_keys']))\n",
    "\n",
    "            media_type = [ media_includes['type'] for media_includes in data['includes']['media'] if media_includes['media_key'] == media_key ][0]\n",
    "            tweets_dic['media_type'].append(media_type)\n",
    "            \n",
    "        else:\n",
    "\n",
    "            media_type_referenced = ''\n",
    "            if author_referenced_tweet != nome_candidato:\n",
    "                media_type_referenced = get_media_referenced(referenced_tweets_id)\n",
    "\n",
    "            if media_type_referenced:\n",
    "                tweets_dic['media_key'].append('')\n",
    "                tweets_dic['media_qty'].append(1)\n",
    "                tweets_dic['media_type'].append(media_type_referenced)\n",
    "            else:\n",
    "                tweets_dic['media_key'].append('')\n",
    "                tweets_dic['media_qty'].append(0)\n",
    "                tweets_dic['media_type'].append('')\n",
    "\n",
    "        if 'entities' in tweet.keys():\n",
    "            mentions = [mention['username'] for mention in tweet['entities']['mentions']]\n",
    "            tweets_dic['mentions'].append(\", \".join(mentions))\n",
    "            tweets_dic['qtd_mencoes'].append(  len(tweet['entities']['mentions'])  )\n",
    "        else:\n",
    "            tweets_dic['mentions'].append(\"\")\n",
    "            tweets_dic['qtd_mencoes'].append(0)\n",
    "\n",
    "\n",
    "    next_token = data['meta'].get('next_token','')\n",
    "\n",
    "    if next_token == \"\":\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.DataFrame(tweets_dic).set_index(\"id\").sort_index()\n",
    "# .sort_values(by='created_at')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1460, 16)"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic_depth = {}\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "     this_depth = 0\n",
    "     if row['referenced_tweets_id']:\n",
    "          if row['referenced_tweets_id'] in dic_depth.keys():\n",
    "               this_depth = dic_depth[row['referenced_tweets_id']] + 1\n",
    "\n",
    "     dic_depth[index] = this_depth\n",
    "\n",
    "df1['thread_count'] = dic_depth.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>like_count</th>\n",
       "      <th>referenced_tweets_id</th>\n",
       "      <th>referenced_tweets_type</th>\n",
       "      <th>referenced_tweets_author</th>\n",
       "      <th>thread_count</th>\n",
       "      <th>media_type</th>\n",
       "      <th>mentions</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1478117280827510795</th>\n",
       "      <td>2022-01-03T21:33:16.000Z</td>\n",
       "      <td>https://t.co/GosonhQxkN</td>\n",
       "      <td>4848</td>\n",
       "      <td>1478117272921194501</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>LulaOficial</td>\n",
       "      <td>1</td>\n",
       "      <td>photo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1478117285919440899</th>\n",
       "      <td>2022-01-03T21:33:17.000Z</td>\n",
       "      <td>https://t.co/GEJWIBW4oc</td>\n",
       "      <td>5540</td>\n",
       "      <td>1478117280827510795</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>LulaOficial</td>\n",
       "      <td>2</td>\n",
       "      <td>photo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1478117291464269826</th>\n",
       "      <td>2022-01-03T21:33:19.000Z</td>\n",
       "      <td>https://t.co/6eqarHYhaU</td>\n",
       "      <td>5105</td>\n",
       "      <td>1478117285919440899</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>LulaOficial</td>\n",
       "      <td>3</td>\n",
       "      <td>photo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1478117298263281667</th>\n",
       "      <td>2022-01-03T21:33:20.000Z</td>\n",
       "      <td>https://t.co/ZRybxe4I34</td>\n",
       "      <td>5463</td>\n",
       "      <td>1478117291464269826</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>LulaOficial</td>\n",
       "      <td>4</td>\n",
       "      <td>photo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1478117304697298948</th>\n",
       "      <td>2022-01-03T21:33:22.000Z</td>\n",
       "      <td>https://t.co/ILGrqYFkRk</td>\n",
       "      <td>4390</td>\n",
       "      <td>1478117298263281667</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>LulaOficial</td>\n",
       "      <td>5</td>\n",
       "      <td>photo</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1539718067189694464</th>\n",
       "      <td>2022-06-22T21:12:48.000Z</td>\n",
       "      <td>As 121 diretrizes, debatidas por representante...</td>\n",
       "      <td>1111</td>\n",
       "      <td>1539718065713254406</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>LulaOficial</td>\n",
       "      <td>6</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1541774112183435264</th>\n",
       "      <td>2022-06-28T13:22:47.000Z</td>\n",
       "      <td>@ricardostuckert Já acompanha o Lula no tiktok...</td>\n",
       "      <td>1678</td>\n",
       "      <td>1541739906657140738</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>LulaOficial</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>ricardostuckert</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1542136363952361472</th>\n",
       "      <td>2022-06-29T13:22:15.000Z</td>\n",
       "      <td>@PauloVieiraReal A gente precisa conversar sob...</td>\n",
       "      <td>15545</td>\n",
       "      <td>1542117802257039364</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>PauloVieiraReal</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>PauloVieiraReal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1542149375710855170</th>\n",
       "      <td>2022-06-29T14:13:57.000Z</td>\n",
       "      <td>@PauloVieiraReal Eu e @JanjaLula vamos aí toma...</td>\n",
       "      <td>7341</td>\n",
       "      <td>1542144661489319937</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>PauloVieiraReal</td>\n",
       "      <td>0</td>\n",
       "      <td>video</td>\n",
       "      <td>PauloVieiraReal, JanjaLula</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1542283686372057089</th>\n",
       "      <td>2022-06-29T23:07:39.000Z</td>\n",
       "      <td>@marinalimax @sogentedobem 🎵🎶 É, vem com você,...</td>\n",
       "      <td>4202</td>\n",
       "      <td>1541909489007398913</td>\n",
       "      <td>replied_to</td>\n",
       "      <td>marinalimax</td>\n",
       "      <td>0</td>\n",
       "      <td>video</td>\n",
       "      <td>marinalimax, sogentedobem</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>177 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   created_at  \\\n",
       "id                                              \n",
       "1478117280827510795  2022-01-03T21:33:16.000Z   \n",
       "1478117285919440899  2022-01-03T21:33:17.000Z   \n",
       "1478117291464269826  2022-01-03T21:33:19.000Z   \n",
       "1478117298263281667  2022-01-03T21:33:20.000Z   \n",
       "1478117304697298948  2022-01-03T21:33:22.000Z   \n",
       "...                                       ...   \n",
       "1539718067189694464  2022-06-22T21:12:48.000Z   \n",
       "1541774112183435264  2022-06-28T13:22:47.000Z   \n",
       "1542136363952361472  2022-06-29T13:22:15.000Z   \n",
       "1542149375710855170  2022-06-29T14:13:57.000Z   \n",
       "1542283686372057089  2022-06-29T23:07:39.000Z   \n",
       "\n",
       "                                                                  text  \\\n",
       "id                                                                       \n",
       "1478117280827510795                            https://t.co/GosonhQxkN   \n",
       "1478117285919440899                            https://t.co/GEJWIBW4oc   \n",
       "1478117291464269826                            https://t.co/6eqarHYhaU   \n",
       "1478117298263281667                            https://t.co/ZRybxe4I34   \n",
       "1478117304697298948                            https://t.co/ILGrqYFkRk   \n",
       "...                                                                ...   \n",
       "1539718067189694464  As 121 diretrizes, debatidas por representante...   \n",
       "1541774112183435264  @ricardostuckert Já acompanha o Lula no tiktok...   \n",
       "1542136363952361472  @PauloVieiraReal A gente precisa conversar sob...   \n",
       "1542149375710855170  @PauloVieiraReal Eu e @JanjaLula vamos aí toma...   \n",
       "1542283686372057089  @marinalimax @sogentedobem 🎵🎶 É, vem com você,...   \n",
       "\n",
       "                     like_count referenced_tweets_id referenced_tweets_type  \\\n",
       "id                                                                            \n",
       "1478117280827510795        4848  1478117272921194501             replied_to   \n",
       "1478117285919440899        5540  1478117280827510795             replied_to   \n",
       "1478117291464269826        5105  1478117285919440899             replied_to   \n",
       "1478117298263281667        5463  1478117291464269826             replied_to   \n",
       "1478117304697298948        4390  1478117298263281667             replied_to   \n",
       "...                         ...                  ...                    ...   \n",
       "1539718067189694464        1111  1539718065713254406             replied_to   \n",
       "1541774112183435264        1678  1541739906657140738             replied_to   \n",
       "1542136363952361472       15545  1542117802257039364             replied_to   \n",
       "1542149375710855170        7341  1542144661489319937             replied_to   \n",
       "1542283686372057089        4202  1541909489007398913             replied_to   \n",
       "\n",
       "                    referenced_tweets_author  thread_count media_type  \\\n",
       "id                                                                      \n",
       "1478117280827510795              LulaOficial             1      photo   \n",
       "1478117285919440899              LulaOficial             2      photo   \n",
       "1478117291464269826              LulaOficial             3      photo   \n",
       "1478117298263281667              LulaOficial             4      photo   \n",
       "1478117304697298948              LulaOficial             5      photo   \n",
       "...                                      ...           ...        ...   \n",
       "1539718067189694464              LulaOficial             6              \n",
       "1541774112183435264              LulaOficial             1              \n",
       "1542136363952361472          PauloVieiraReal             0              \n",
       "1542149375710855170          PauloVieiraReal             0      video   \n",
       "1542283686372057089              marinalimax             0      video   \n",
       "\n",
       "                                       mentions  \n",
       "id                                               \n",
       "1478117280827510795                              \n",
       "1478117285919440899                              \n",
       "1478117291464269826                              \n",
       "1478117298263281667                              \n",
       "1478117304697298948                              \n",
       "...                                         ...  \n",
       "1539718067189694464                              \n",
       "1541774112183435264             ricardostuckert  \n",
       "1542136363952361472             PauloVieiraReal  \n",
       "1542149375710855170  PauloVieiraReal, JanjaLula  \n",
       "1542283686372057089   marinalimax, sogentedobem  \n",
       "\n",
       "[177 rows x 9 columns]"
      ]
     },
     "execution_count": 298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1[df1['referenced_tweets_type']=='replied_to'][['created_at', 'text', 'like_count', 'referenced_tweets_id', 'referenced_tweets_type', 'referenced_tweets_author', 'thread_count', 'media_type', 'mentions']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic_ext_links = {}\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "     text = row['text']\n",
    "     # Misturas de quote com media type pode não pegar direito...\n",
    "     if (\"https://\" in text or \"https://\" in  text) and (not text.startswith(\"http\") ) and row['media_type'] == '' and row['referenced_tweets_type'] != 'quoted':\n",
    "          dic_ext_links[index] = 1\n",
    "     else:\n",
    "          dic_ext_links[index] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['ext_links'] = dic_ext_links.values()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic_tipo_tweet = {}\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "     if row['referenced_tweets_type'] == \"\":\n",
    "          dic_tipo_tweet[index] = 'root'\n",
    "     elif row['referenced_tweets_type'] == \"replied_to\" and row['referenced_tweets_author'] != nome_candidato  :\n",
    "          dic_tipo_tweet[index] = 'replied_to_other'\n",
    "     elif row['referenced_tweets_type'] == \"replied_to\" and row['referenced_tweets_author'] == nome_candidato  :\n",
    "          dic_tipo_tweet[index] = 'replied_to_self'\n",
    "     else:\n",
    "          dic_tipo_tweet[index] = row['referenced_tweets_type']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['tweet_type'] = dic_tipo_tweet.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "root                1182\n",
       "replied_to_self      153\n",
       "retweeted             62\n",
       "quoted                39\n",
       "replied_to_other      24\n",
       "Name: tweet_type, dtype: int64"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1['tweet_type'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic_qtd_mencoes = {}\n",
    "\n",
    "for index, row in df1.iterrows():\n",
    "     if row['referenced_tweets_type'] == \"\":\n",
    "          dic_qtd_mencoes[index] = 'root'\n",
    "     elif row['referenced_tweets_type'] == \"replied_to\" and row['referenced_tweets_author'] != nome_candidato  :\n",
    "          dic_qtd_mencoes[index] = 'replied_to_other'\n",
    "     elif row['referenced_tweets_type'] == \"replied_to\" and row['referenced_tweets_author'] == nome_candidato  :\n",
    "          dic_qtd_mencoes[index] = 'replied_to_self'\n",
    "     else:\n",
    "          dic_qtd_mencoes[index] = row['referenced_tweets_type']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df1.to_pickle(\"./bolsonaro_2022_Q12.pkl\")\n",
    "df1.to_pickle(\"./lula_2022_Q12.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 64-bit (microsoft store)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "593ca7ad6d5b66771a0a8bb07c067b8882d1821c8e891fd5790867f02ea2a8c2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}