farm_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import urllib\n",
    "import re "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(query):\n",
    "    address = \"http://www.bing.com/search?q=%s\" % (urllib.parse.quote_plus(query)+ \"&count=20\")\n",
    "\n",
    "    getRequest = urllib.request.Request(address, None, {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0'})\n",
    "\n",
    "    urlfile = urllib.request.urlopen(getRequest)\n",
    "    htmlResult = urlfile.read(20000000)\n",
    "    urlfile.close()\n",
    "\n",
    "    soup = BeautifulSoup(htmlResult).find('div',id='b_content')\n",
    "\n",
    "    [s.extract() for s in soup('span')]\n",
    "    unwantedTags = ['cite']\n",
    "    \n",
    "    for tag in unwantedTags:\n",
    "        for match in soup.findAll(tag):\n",
    "            match.replaceWithChildren()\n",
    "\n",
    "    results = soup.findAll('li')\n",
    "    \n",
    "\n",
    "    return results\n",
    "\n",
    "def get_item_info(query):\n",
    "    links = search(query)\n",
    "    titles = []\n",
    "    urls = []\n",
    "    paragraphs = []\n",
    "    for link in links :\n",
    "        try:\n",
    "            urls.append(str(link.find('h2').find('a')['href']).replace(\" \", \" \"))\n",
    "            titles.append(str(link.find('h2').text).replace(\" \", \" \"))\n",
    "        except:\n",
    "            \"\"\n",
    "        try:\n",
    "            paragraphs.append(str(link.findAll('p')[0].text))\n",
    "        except:\n",
    "            paragraphs.append(\"\")\n",
    "\n",
    "    return titles, urls, paragraphs, links[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_water_data(item):\n",
    "    titles, urls, paragraphs, b = get_item_info(\"how much do I need to water my \"+item+\" plant\")\n",
    "    i = 0\n",
    "    for p in paragraphs:\n",
    "        nums = parse_water_paragraph(p)\n",
    "        if len(nums)==2:\n",
    "            print(nums)\n",
    "            return nums\n",
    "    return [1,10]\n",
    "\n",
    "def parse_water_paragraph(p):\n",
    "    tmp = \"\"\n",
    "    if 'inch' in p:\n",
    "        tmp = p[(p.index('inch')-15):(p.index('inch'))]\n",
    "    \n",
    "    nums = re.findall('\\d*\\.?\\d+',tmp)\n",
    "    print(nums)\n",
    "    return nums\n",
    "\n",
    "def get_growth_data(item):\n",
    "    results = search(\"how long does it take for a \"+item+\" to grow in days\")\n",
    "    ans = []\n",
    "    for r in results:\n",
    "        ans = parse_growth_paragraph(str(r))\n",
    "        if len(ans)>0:\n",
    "            return ans\n",
    "\n",
    "    return ans\n",
    "\n",
    "def parse_growth_paragraph(p):\n",
    "    tmp = \"\"\n",
    "    month = False\n",
    "    if 'day' in p:\n",
    "        print(\"Hello\")\n",
    "        tmp = p[(p.index('day')-15):(p.index('day'))]\n",
    "\n",
    "    nums = re.findall('\\d*\\.?\\d+',tmp)\n",
    "            \n",
    "    return nums\n",
    "\n",
    "def get_expiration_data(item):\n",
    "    results = search(\"how long does it take for a \"+item+\" to expire\")\n",
    "    ans = []\n",
    "    for r in results:\n",
    "        ans = parse_expiration_paragraph(str(r))\n",
    "        if len(ans)>0:\n",
    "            return ans\n",
    "\n",
    "    return ans\n",
    "\n",
    "def parse_expiration_paragraph(p):\n",
    "    tmp = \"\"\n",
    "    month = False\n",
    "    if 'day' in p:\n",
    "        print(\"Hello\")\n",
    "        tmp = p[(p.index('day')-15):(p.index('day'))]\n",
    "    elif 'week' in p:\n",
    "        tmp = \"7\"\n",
    "\n",
    "    nums = re.findall('\\d*\\.?\\d+',tmp)\n",
    "            \n",
    "    return nums\n",
    "\n",
    "\n",
    "def get_shop_data(item):\n",
    "    shopdata = shop(\"Buy \"+item+\" seeds\")\n",
    "    shopdata = []\n",
    "    for l in links:\n",
    "        data = {}\n",
    "        try:\n",
    "            data['image']='https://www.bing.com/'+l.findAll('img')[0]['src']\n",
    "            data['seller'] = l.findAll('span', class_='br-sellersCite')[0].text\n",
    "            data['price'] = l.findAll('div', class_='pd-price br-standardPrice promoted')[0].text\n",
    "            try:\n",
    "                data['item'] = l.findAll('span')[-1]['title']\n",
    "            except:\n",
    "                data['item'] = \"shoes\"\n",
    "            data['url'] = l.findAll('a', class_='br-adClickOut')[0]['href']\n",
    "\n",
    "            shopdata.append(data)\n",
    "        except:\n",
    "            print('DIDNT WORK')\n",
    "            break\n",
    "    return shopdata\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "['1', '2']\n",
      "['1', '2']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['1', '2']"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_water_data(\"daisy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['7']"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_expiration_data('potato')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}