google_news_headlines_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import  requests \n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "from urllib.error import URLError, HTTPError\n",
    "from urllib3.exceptions import SSLError\n",
    "\n",
    "from requests.exceptions import HTTPError\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import urlopen, Request\n",
    "from socket import timeout\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://news.google.com/search?q=turkish+airlines\n"
     ]
    }
   ],
   "source": [
    "search_input = 'Turkish Airlines'\n",
    "search_string = '+'.join(str(e) for e in search_input.lower().split())\n",
    "url= f'https://news.google.com/search?q={search_string}'\n",
    "print(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_time_range(data):\n",
    "    \n",
    "    minm = data[0][1][0]\n",
    "    maxm = data[0][1][0]\n",
    "    for i in data:\n",
    "        ptr = i[1][0].split('-')\n",
    "        minm2 = minm.split('-')\n",
    "        maxm2 = maxm.split('-')\n",
    "        \n",
    "        if int(ptr[0]) > int(maxm2[0]):\n",
    "            maxm = '-'.join(str(e) for e in ptr)\n",
    "        if int(ptr[0]) < int(minm2[0]):\n",
    "            minm = '-'.join(str(e) for e in ptr)\n",
    "            \n",
    "        if  int(ptr[0]) == int(maxm2[0]):\n",
    "            \n",
    "            mon_ptr= int(ptr[1].replace('0', ''))\n",
    "            \n",
    "            mon_maxm2 = int(maxm2[1].replace('0', ''))\n",
    "            mon_minm2 = int(minm2[1].replace('0', ''))\n",
    "                        \n",
    "            if mon_ptr > mon_maxm2:\n",
    "                maxm = '-'.join(str(e) for e in ptr)\n",
    "                \n",
    "            if mon_ptr < mon_minm2:\n",
    "                minm = '-'.join(str(e) for e in ptr)\n",
    "            \n",
    "            elif  int(mon_ptr) == int(mon_maxm2):\n",
    "                \n",
    "                day_ptr= int(ptr[2].replace('0', ''))\n",
    "            \n",
    "                day_maxm2 = int(maxm2[2].replace('0', ''))\n",
    "                day_minm2 = int(minm2[2].replace('0', ''))\n",
    "                \n",
    "                            \n",
    "                if day_ptr > day_maxm2:\n",
    "                    maxm = '-'.join(str(e) for e in ptr)\n",
    "                \n",
    "                if day_ptr < day_minm2:\n",
    "                    minm = '-'.join(str(e) for e in ptr)\n",
    "                \n",
    "            \n",
    "            \n",
    "    return [minm,maxm]\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104\n"
     ]
    }
   ],
   "source": [
    "headlines = []\n",
    "\n",
    "try: \n",
    "    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}\n",
    "    reg_url = url\n",
    "    req = Request(url=reg_url, headers=headers) \n",
    "    html = urlopen(req).read()  \n",
    "\n",
    "    soup = BeautifulSoup(html, \"html.parser\")\n",
    "    \n",
    "    data = soup.findAll('article')\n",
    "    print(len(data))\n",
    "    for news in data:\n",
    "        headline = str(news).split('</a')[1].split('\">')[2]\n",
    "        if 'datetime' in str(news):\n",
    "            time = str(news).split('datetime=\"')[1].split('\">')[0].split('T')\n",
    "            headlines.append([headline,time])\n",
    "\n",
    "    \n",
    "except HTTPError as http_err:\n",
    "    print(f'HTTP error occurred: {http_err}')  \n",
    "    pass\n",
    "except Exception as err:\n",
    "    print(f'Other1 error occurred: {err}')  \n",
    "    pass\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source: Google News Headlines\n",
      "Searched Keyword: Turkish Airlines\n",
      "104 News Headlines Found\n",
      "oldest headline date:  2019-10-12\n",
      "recent headline date:  2020-03-28\n"
     ]
    }
   ],
   "source": [
    "rang = check_time_range(headlines); minm = rang[0];maxm=rang[1]\n",
    "\n",
    "\n",
    "print('Source: Google News Headlines')\n",
    "print(f'Searched Keyword: {search_input}') \n",
    "print(f'{len(headlines)} News Headlines Found')\n",
    "print('oldest headline date: ',minm)\n",
    "print('recent headline date: ',maxm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Turkish Airlines suspends flights to New York</td>\n",
       "      <td>2020-03-27 01:46:57Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Turkish Airlines Cuts New York Flights Due To ...</td>\n",
       "      <td>2020-03-27 15:48:40Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Turkish Airlines suspends flights to New York ...</td>\n",
       "      <td>2020-03-27 10:10:54Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Turkish Airlines Cancels Flights to New York</td>\n",
       "      <td>2020-03-27 06:29:46Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Turkish Airlines Suspends New York and Moscow ...</td>\n",
       "      <td>2020-03-27 13:01:51Z</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text                  date\n",
       "0      Turkish Airlines suspends flights to New York  2020-03-27 01:46:57Z\n",
       "1  Turkish Airlines Cuts New York Flights Due To ...  2020-03-27 15:48:40Z\n",
       "2  Turkish Airlines suspends flights to New York ...  2020-03-27 10:10:54Z\n",
       "3       Turkish Airlines Cancels Flights to New York  2020-03-27 06:29:46Z\n",
       "4  Turkish Airlines Suspends New York and Moscow ...  2020-03-27 13:01:51Z"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headlines_ = []\n",
    "for j in headlines:\n",
    "    temp = j[1][0] +' '+j[1][1]\n",
    "    headlines_.append([j[0],temp])\n",
    "    \n",
    "df = pd.DataFrame(headlines_, columns =['text','date'])\n",
    "df.to_csv(f'apis_datasets/{search_input}_GoogleNewsHeadlines.csv', encoding='utf-8', index=False)\n",
    "\n",
    "df.head()     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}