removeDate_RE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:38:56.817647Z",
     "start_time": "2018-03-04T21:38:54.915937Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import re\n",
    "\n",
    "train = pd.read_csv('data/train.csv')\n",
    "test = pd.read_csv('data/test.csv')\n",
    "pd.set_option('display.max_rows', 40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:38:56.823592Z",
     "start_time": "2018-03-04T21:38:56.819080Z"
    }
   },
   "outputs": [],
   "source": [
    "def findCombined(txt):\n",
    "    myResults = []\n",
    "    for matchGroup in myDate.finditer(txt):\n",
    "        myResults.append(matchGroup.group('combined'))\n",
    "    return myResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:38:56.955828Z",
     "start_time": "2018-03-04T21:38:56.827106Z"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "tqdm.pandas(tqdm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:38:57.073939Z",
     "start_time": "2018-03-04T21:38:56.957339Z"
    }
   },
   "outputs": [],
   "source": [
    "nDAY = r'(?:[0-3]?\\d)'  # day can be from 1 to 31 with a leading zero \n",
    "nMNTH = r'(?:11|12|10|0?[1-9])' # month can be 1 to 12 with a leading zero\n",
    "nYR = r'(?:(?:19|20)\\d\\d)'  # I've restricted the year to being in 20th or 21st century on the basis \n",
    "                            # that people doon't generally use all number format for old dates, but write them out \n",
    "nDELIM = r'(?:[\\/\\-\\._])?'  # \n",
    "NUM_DATE = f\"\"\"\n",
    "    (?P<num_date>\n",
    "        (?:^|\\D) # new bit here\n",
    "        (?:\n",
    "        # YYYY-MM-DD\n",
    "        (?:{nYR}(?P<delim1>[\\/\\-\\._]?){nMNTH}(?P=delim1){nDAY})\n",
    "        |\n",
    "        # YYYY-DD-MM\n",
    "        (?:{nYR}(?P<delim2>[\\/\\-\\._]?){nDAY}(?P=delim2){nMNTH})\n",
    "        |\n",
    "        # DD-MM-YYYY\n",
    "        (?:{nDAY}(?P<delim3>[\\/\\-\\._]?){nMNTH}(?P=delim3){nYR})\n",
    "        |\n",
    "        # MM-DD-YYYY\n",
    "        (?:{nMNTH}(?P<delim4>[\\/\\-\\._]?){nDAY}(?P=delim4){nYR})\n",
    "        )\n",
    "        (?:\\D|$) # new bit here\n",
    "    )\"\"\"\n",
    "DAY = r\"\"\"\n",
    "(?:\n",
    "    # search 1st 2nd 3rd etc, or first second third\n",
    "    (?:[23]?1st|2{1,2}nd|\\d{1,2}th|2?3rd|first|second|third|fourth|fifth|sixth|seventh|eighth|nineth)\n",
    "    |\n",
    "    # or just a number, but without a leading zero\n",
    "    (?:[123]?\\d)\n",
    ")\"\"\"\n",
    "MONTH = r'(?:january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)'\n",
    "YEAR = r\"\"\"(?:(?:[12]?\\d|')?\\d\\d)\"\"\"\n",
    "DELIM = r'(?:\\s*(?:[\\s\\.\\-\\\\/,]|(?:of))\\s*)'\n",
    "\n",
    "YEAR_4D = r\"\"\"(?:[12]\\d\\d\\d)\"\"\"\n",
    "DATE_PATTERN = f\"\"\"(?P<wordy_date>\n",
    "    # non word character or start of string\n",
    "    (?:^|\\W)\n",
    "        (?:\n",
    "            # match various combinations of year month and day \n",
    "            (?:\n",
    "                # 4 digit year\n",
    "                (?:{YEAR_4D}{DELIM})?\n",
    "                    (?:\n",
    "                    # Day - Month\n",
    "                    (?:{DAY}{DELIM}{MONTH})\n",
    "                    |\n",
    "                    # Month - Day\n",
    "                    (?:{MONTH}{DELIM}{DAY})\n",
    "                    )\n",
    "                # 2 or 4 digit year\n",
    "                (?:{DELIM}{YEAR})?\n",
    "            )\n",
    "            |\n",
    "            # Month - Year (2 or 3 digit)\n",
    "            (?:{MONTH}{DELIM}{YEAR})\n",
    "        )\n",
    "    # non-word character or end of string\n",
    "    (?:$|\\W)\n",
    ")\"\"\"\n",
    "\n",
    "TIME = r\"\"\"(?:\n",
    "(?:\n",
    "# first number should be 0 - 59 with optional leading zero.\n",
    "[012345]?\\d\n",
    "# second number is the same following a colon\n",
    ":[012345]\\d\n",
    ")\n",
    "# next we add our optional seconds number in the same format\n",
    "(?::[012345]\\d)?\n",
    "# and finally add optional am or pm possibly with . and spaces\n",
    "(?:\\s*(?:a|p)\\.?m\\.?)?\n",
    ")\"\"\"\n",
    "\n",
    "COMBINED = f\"\"\"(?P<combined>\n",
    "    (?:\n",
    "        # time followed by date, or date followed by time\n",
    "        {TIME}?{DATE_PATTERN}{TIME}?\n",
    "        |\n",
    "        # or as above but with the numeric version of the date\n",
    "        {TIME}?{NUM_DATE}{TIME}?\n",
    "    ) \n",
    "    # or a time on its own\n",
    "    |\n",
    "    (?:{TIME})\n",
    ")\"\"\"\n",
    "\n",
    "myDate = re.compile(COMBINED, re.IGNORECASE | re.VERBOSE | re.UNICODE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:39:04.914280Z",
     "start_time": "2018-03-04T21:38:57.075444Z"
    }
   },
   "outputs": [],
   "source": [
    "for comments in train,test:\n",
    "    # remove '\\\\n'\n",
    "    comments['comment_text'] = comments['comment_text'].map(lambda x: re.sub('\\\\n',' ',str(x)))\n",
    "    \n",
    "    # remove any text starting with User... \n",
    "    comments['comment_text'] = comments['comment_text'].map(lambda x: re.sub(\"\\[\\[User.*\",'',str(x)))\n",
    "    \n",
    "    # remove IP addresses or user IDs\n",
    "    comments['comment_text'] = comments['comment_text'].map(lambda x: re.sub(\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",'',str(x)))\n",
    "    \n",
    "    # lower uppercase letters\n",
    "    comments['comment_text'] = comments['comment_text'].map(lambda x: str(x).lower())\n",
    "    \n",
    "    #remove http links in the text\n",
    "    comments['comment_text'] = comments['comment_text'].map(lambda x: re.sub(\"(http://.*?\\s)|(http://.*)\",'',str(x)))\n",
    "    \n",
    "    #remove all punctuation except for apostrophe (')\n",
    "    #comments['comment_text'] = comments['comment_text'].map(lambda x: re.sub('[!\"#$%&\\()*+,-./:;<=>?@[\\\\]^_`{|}~]','',str(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:40:45.014309Z",
     "start_time": "2018-03-04T21:39:04.916225Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████| 159571/159571 [00:52<00:00, 3063.19it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 153164/153164 [00:47<00:00, 3193.25it/s]\n"
     ]
    }
   ],
   "source": [
    "for df in [train, test]:\n",
    "    df['fewer_dates'] = df.comment_text.progress_apply(lambda x: myDate.sub(' xxDATExx ', x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:40:45.300572Z",
     "start_time": "2018-03-04T21:40:45.018320Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 12667 rows with dates in the training set\n",
      "Found 9098 rows with dates in the test set\n"
     ]
    }
   ],
   "source": [
    "print('Found {} rows with dates in the training set'.format(train.fewer_dates.str.contains('xxDATExx').sum()))\n",
    "print('Found {} rows with dates in the test set'.format(test.fewer_dates.str.contains('xxDATExx').sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:40:45.495115Z",
     "start_time": "2018-03-04T21:40:45.303077Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment_text</th>\n",
       "      <th>fewer_dates</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d'aww! he matches this background colour i'm seemingly stuck with. thanks.  (talk) 21:51, january 11, 2016 (utc)</td>\n",
       "      <td>d'aww! he matches this background colour i'm seemingly stuck with. thanks.  (talk)  xxDATExx , xxDATExx (utc)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>\" fair use rationale for image:wonju.jpg  thanks for uploading image:wonju.jpg. i notice the image page specifies that the image is being used under fair use but there is no explanation or rationale as to why its use in wikipedia articles constitutes fair use. in addition to the boilerplate fair use template, you must also write out on the image description page a specific explanation or rationale for why using this image in each article is consistent with fair use.  please go to the image description page and edit it to include a fair use rationale.  if you have uploaded other fair use media, consider checking that you have specified the fair use rationale on those pages too. you can find a list of 'image' pages you have edited by clicking on the \"\"my contributions\"\" link (it is located at the very top of any wikipedia page when you are logged in), and then selecting \"\"image\"\" from the dropdown box. note that any fair use images uploaded after 4 may, 2006, and lacking such an explanation will be deleted one week after they have been uploaded, as described on criteria for speedy deletion. if you have any questions please ask them at the media copyright questions page. thank you. (talk • contribs • )  unspecified source for image:wonju.jpg  thanks for uploading image:wonju.jpg. i noticed that the file's description page currently doesn't specify who created the content, so the copyright status is unclear. if you did not create this file yourself, then you will need to specify the owner of the copyright. if you obtained it from a website, then a link to the website from which it was taken, together with a restatement of that website's terms of use of its content, is usually sufficient information. however, if the copyright holder is different from the website's publisher, then their copyright should also be acknowledged.  as well as adding the source, please add a proper copyright licensing tag if the file doesn't have one already. if you created/took the picture, audio, or video then the  tag can be used to release it under the gfdl. if you believe the media meets the criteria at wikipedia:fair use, use a tag such as  or one of the other tags listed at wikipedia:image copyright tags#fair use. see wikipedia:image copyright tags for the full list of copyright tags that you can use.  if you have uploaded other files, consider checking that you have specified their source and tagged them, too. you can find a list of files you have uploaded by following [ this link]. unsourced and untagged images may be deleted one week after they have been tagged, as described on criteria for speedy deletion. if the image is copyrighted under a non-free license (per wikipedia:fair use) then the image will be deleted 48 hours after . if you have any questions please ask them at the media copyright questions page. thank you. (talk • contribs • ) \"</td>\n",
       "      <td>\" fair use rationale for image:wonju.jpg  thanks for uploading image:wonju.jpg. i notice the image page specifies that the image is being used under fair use but there is no explanation or rationale as to why its use in wikipedia articles constitutes fair use. in addition to the boilerplate fair use template, you must also write out on the image description page a specific explanation or rationale for why using this image in each article is consistent with fair use.  please go to the image description page and edit it to include a fair use rationale.  if you have uploaded other fair use media, consider checking that you have specified the fair use rationale on those pages too. you can find a list of 'image' pages you have edited by clicking on the \"\"my contributions\"\" link (it is located at the very top of any wikipedia page when you are logged in), and then selecting \"\"image\"\" from the dropdown box. note that any fair use images uploaded after xxDATExx  and lacking such an explanation will be deleted one week after they have been uploaded, as described on criteria for speedy deletion. if you have any questions please ask them at the media copyright questions page. thank you. (talk • contribs • )  unspecified source for image:wonju.jpg  thanks for uploading image:wonju.jpg. i noticed that the file's description page currently doesn't specify who created the content, so the copyright status is unclear. if you did not create this file yourself, then you will need to specify the owner of the copyright. if you obtained it from a website, then a link to the website from which it was taken, together with a restatement of that website's terms of use of its content, is usually sufficient information. however, if the copyright holder is different from the website's publisher, then their copyright should also be acknowledged.  as well as adding the source, please add a proper copyright licensing tag if the file doesn't have one already. if you created/took the picture, audio, or video then the  tag can be used to release it under the gfdl. if you believe the media meets the criteria at wikipedia:fair use, use a tag such as  or one of the other tags listed at wikipedia:image copyright tags#fair use. see wikipedia:image copyright tags for the full list of copyright tags that you can use.  if you have uploaded other files, consider checking that you have specified their source and tagged them, too. you can find a list of files you have uploaded by following [ this link]. unsourced and untagged images may be deleted one week after they have been tagged, as described on criteria for speedy deletion. if the image is copyrighted under a non-free license (per wikipedia:fair use) then the image will be deleted 48 hours after . if you have any questions please ask them at the media copyright questions page. thank you. (talk • contribs • ) \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>\"  juelz santanas age  in 2002, juelz santana was 18 years old, then came february 18th, which makes juelz turn 19 making songs with the diplomats. the third neff to be signed to cam's label under roc a fella. in 2003, he was 20 years old coming out with his own singles \"\"santana's town\"\" and \"\"down\"\". so yes, he is born in 1983. he really is, how could he be older then lloyd banks? and how could he be 22 when his birthday passed? the homie neff is 23 years old. 1983 - 2006 (juelz death, god forbid if your thinking about that) equals 23. go to your caculator and stop changing his year of birth. my god.\"</td>\n",
       "      <td>\"  juelz santanas age  in 2002, juelz santana was 18 years old, then came xxDATExx  which makes juelz turn 19 making songs with the diplomats. the third neff to be signed to cam's label under roc a fella. in 2003, he was 20 years old coming out with his own singles \"\"santana's town\"\" and \"\"down\"\". so yes, he is born in 1983. he really is, how could he be older then lloyd banks? and how could he be 22 when his birthday passed? the homie neff is 23 years old. 1983 - 2006 (juelz death, god forbid if your thinking about that) equals 23. go to your caculator and stop changing his year of birth. my god.\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>\"   the signpost: 24 september 2012    read this signpost in full  single-page  unsubscribe     \"</td>\n",
       "      <td>\"   the signpost: xxDATExx    read this signpost in full  single-page  unsubscribe     \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>yes, because the mother of the child in the case against michael jackson was studied in here motives and reasonings and judged upon her character just as harshly as wacko jacko himself.  don't tell me to ignore it and incriminate myself.  i am going to continue refuting the bullshit that jayjg keeps throwing at me.   18:01, 16 jun 2005 (utc)</td>\n",
       "      <td>yes, because the mother of the child in the case against michael jackson was studied in here motives and reasonings and judged upon her character just as harshly as wacko jacko himself.  don't tell me to ignore it and incriminate myself.  i am going to continue refuting the bullshit that jayjg keeps throwing at me.    xxDATExx , xxDATExx (utc)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   comment_text  \\\n",
       "1   d'aww! he matches this background colour i'm seemingly stuck with. thanks.  (talk) 21:51, january 11, 2016 (utc)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
       "10  \" fair use rationale for image:wonju.jpg  thanks for uploading image:wonju.jpg. i notice the image page specifies that the image is being used under fair use but there is no explanation or rationale as to why its use in wikipedia articles constitutes fair use. in addition to the boilerplate fair use template, you must also write out on the image description page a specific explanation or rationale for why using this image in each article is consistent with fair use.  please go to the image description page and edit it to include a fair use rationale.  if you have uploaded other fair use media, consider checking that you have specified the fair use rationale on those pages too. you can find a list of 'image' pages you have edited by clicking on the \"\"my contributions\"\" link (it is located at the very top of any wikipedia page when you are logged in), and then selecting \"\"image\"\" from the dropdown box. note that any fair use images uploaded after 4 may, 2006, and lacking such an explanation will be deleted one week after they have been uploaded, as described on criteria for speedy deletion. if you have any questions please ask them at the media copyright questions page. thank you. (talk • contribs • )  unspecified source for image:wonju.jpg  thanks for uploading image:wonju.jpg. i noticed that the file's description page currently doesn't specify who created the content, so the copyright status is unclear. if you did not create this file yourself, then you will need to specify the owner of the copyright. if you obtained it from a website, then a link to the website from which it was taken, together with a restatement of that website's terms of use of its content, is usually sufficient information. however, if the copyright holder is different from the website's publisher, then their copyright should also be acknowledged.  as well as adding the source, please add a proper copyright licensing tag if the file doesn't have one already. if you created/took the picture, audio, or video then the  tag can be used to release it under the gfdl. if you believe the media meets the criteria at wikipedia:fair use, use a tag such as  or one of the other tags listed at wikipedia:image copyright tags#fair use. see wikipedia:image copyright tags for the full list of copyright tags that you can use.  if you have uploaded other files, consider checking that you have specified their source and tagged them, too. you can find a list of files you have uploaded by following [ this link]. unsourced and untagged images may be deleted one week after they have been tagged, as described on criteria for speedy deletion. if the image is copyrighted under a non-free license (per wikipedia:fair use) then the image will be deleted 48 hours after . if you have any questions please ask them at the media copyright questions page. thank you. (talk • contribs • ) \"   \n",
       "15  \"  juelz santanas age  in 2002, juelz santana was 18 years old, then came february 18th, which makes juelz turn 19 making songs with the diplomats. the third neff to be signed to cam's label under roc a fella. in 2003, he was 20 years old coming out with his own singles \"\"santana's town\"\" and \"\"down\"\". so yes, he is born in 1983. he really is, how could he be older then lloyd banks? and how could he be 22 when his birthday passed? the homie neff is 23 years old. 1983 - 2006 (juelz death, god forbid if your thinking about that) equals 23. go to your caculator and stop changing his year of birth. my god.\"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
       "23  \"   the signpost: 24 september 2012    read this signpost in full  single-page  unsubscribe     \"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
       "27  yes, because the mother of the child in the case against michael jackson was studied in here motives and reasonings and judged upon her character just as harshly as wacko jacko himself.  don't tell me to ignore it and incriminate myself.  i am going to continue refuting the bullshit that jayjg keeps throwing at me.   18:01, 16 jun 2005 (utc)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 fewer_dates  \n",
       "1   d'aww! he matches this background colour i'm seemingly stuck with. thanks.  (talk)  xxDATExx , xxDATExx (utc)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
       "10  \" fair use rationale for image:wonju.jpg  thanks for uploading image:wonju.jpg. i notice the image page specifies that the image is being used under fair use but there is no explanation or rationale as to why its use in wikipedia articles constitutes fair use. in addition to the boilerplate fair use template, you must also write out on the image description page a specific explanation or rationale for why using this image in each article is consistent with fair use.  please go to the image description page and edit it to include a fair use rationale.  if you have uploaded other fair use media, consider checking that you have specified the fair use rationale on those pages too. you can find a list of 'image' pages you have edited by clicking on the \"\"my contributions\"\" link (it is located at the very top of any wikipedia page when you are logged in), and then selecting \"\"image\"\" from the dropdown box. note that any fair use images uploaded after xxDATExx  and lacking such an explanation will be deleted one week after they have been uploaded, as described on criteria for speedy deletion. if you have any questions please ask them at the media copyright questions page. thank you. (talk • contribs • )  unspecified source for image:wonju.jpg  thanks for uploading image:wonju.jpg. i noticed that the file's description page currently doesn't specify who created the content, so the copyright status is unclear. if you did not create this file yourself, then you will need to specify the owner of the copyright. if you obtained it from a website, then a link to the website from which it was taken, together with a restatement of that website's terms of use of its content, is usually sufficient information. however, if the copyright holder is different from the website's publisher, then their copyright should also be acknowledged.  as well as adding the source, please add a proper copyright licensing tag if the file doesn't have one already. if you created/took the picture, audio, or video then the  tag can be used to release it under the gfdl. if you believe the media meets the criteria at wikipedia:fair use, use a tag such as  or one of the other tags listed at wikipedia:image copyright tags#fair use. see wikipedia:image copyright tags for the full list of copyright tags that you can use.  if you have uploaded other files, consider checking that you have specified their source and tagged them, too. you can find a list of files you have uploaded by following [ this link]. unsourced and untagged images may be deleted one week after they have been tagged, as described on criteria for speedy deletion. if the image is copyrighted under a non-free license (per wikipedia:fair use) then the image will be deleted 48 hours after . if you have any questions please ask them at the media copyright questions page. thank you. (talk • contribs • ) \"  \n",
       "15  \"  juelz santanas age  in 2002, juelz santana was 18 years old, then came xxDATExx  which makes juelz turn 19 making songs with the diplomats. the third neff to be signed to cam's label under roc a fella. in 2003, he was 20 years old coming out with his own singles \"\"santana's town\"\" and \"\"down\"\". so yes, he is born in 1983. he really is, how could he be older then lloyd banks? and how could he be 22 when his birthday passed? the homie neff is 23 years old. 1983 - 2006 (juelz death, god forbid if your thinking about that) equals 23. go to your caculator and stop changing his year of birth. my god.\"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
       "23  \"   the signpost: xxDATExx    read this signpost in full  single-page  unsubscribe     \"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n",
       "27  yes, because the mother of the child in the case against michael jackson was studied in here motives and reasonings and judged upon her character just as harshly as wacko jacko himself.  don't tell me to ignore it and incriminate myself.  i am going to continue refuting the bullshit that jayjg keeps throwing at me.    xxDATExx , xxDATExx (utc)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', -1)\n",
    "train.loc[train.fewer_dates.str.contains('xxDATExx'), ['comment_text', 'fewer_dates']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:40:51.899590Z",
     "start_time": "2018-03-04T21:40:45.497093Z"
    }
   },
   "outputs": [],
   "source": [
    "train.to_csv('data/train_nodate.csv', index=False)\n",
    "test.to_csv('data/test_nodate.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-04T21:32:37.904968Z",
     "start_time": "2018-03-04T21:30:49.103Z"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}