simplifyweibo_4_moods数据集练习.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n作者：英俊\\nQQ:2227495940\\n邮箱 同上\\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "作者：英俊\n",
    "QQ:2227495940\n",
    "邮箱 同上\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>﻿啊呀呀！要死啦！么么么！只穿外套就好了，我认为里面那件很多余啊周小伦喜歡 你各種 五角星的...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>嗯……既然大姚通知了……那我也表示下收到……姚，你知道吗？假如外星人入侵地球，只要摧毁我们的...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>风格不一样嘛，都喜欢！最喜欢哪张？</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>好呀，试试D .I .Y .去死皮面膜1.将燕麦片加水中浸泡6小时，加入木瓜牛奶搅拌。2.放...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>张老师，谢谢侬的信任！粉丝多少无所谓重在质地近日发现一个现象——他加了你关注，你回加后，他立...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                             review\n",
       "0      0  ﻿啊呀呀！要死啦！么么么！只穿外套就好了，我认为里面那件很多余啊周小伦喜歡 你各種 五角星的...\n",
       "1      0  嗯……既然大姚通知了……那我也表示下收到……姚，你知道吗？假如外星人入侵地球，只要摧毁我们的...\n",
       "2      0                                  风格不一样嘛，都喜欢！最喜欢哪张？\n",
       "3      0  好呀，试试D .I .Y .去死皮面膜1.将燕麦片加水中浸泡6小时，加入木瓜牛奶搅拌。2.放...\n",
       "4      0  张老师，谢谢侬的信任！粉丝多少无所谓重在质地近日发现一个现象——他加了你关注，你回加后，他立..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df_all=pd.read_csv(r'./simplifyweibo_4_moods/simplifyweibo_4_moods.csv')\n",
    "df_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_all.isnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_train['label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_dev['label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_test['label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_train.isnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, 3], dtype=int64)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all['label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all=df_all.sample(8000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all1=pd.DataFrame({\"raw_words\":df_all['review'],\"target\":df_all['label']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw_words</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>282418</th>\n",
       "      <td>元宝路？据说，谁看到并这张图片，就会“钱途”似锦，不想发财都难！</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154207</th>\n",
       "      <td>自作聪明的人，不在少数，尤其现在社会中。自作聪明，反被聪明误；自以为是，必自作自受。不以为然...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292698</th>\n",
       "      <td>悼念930事件。各位一路走好。。。。</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210638</th>\n",
       "      <td>中国人为什么自愿做房奴？难道租房比一辈子还贷更让人无法接受？</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229423</th>\n",
       "      <td>强烈要求真相！！！。是哦～闹鬼也不太现实，我倒宁可相信这个是真相连续死亡的10位工人不是自杀...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                raw_words  target\n",
       "282418                   元宝路？据说，谁看到并这张图片，就会“钱途”似锦，不想发财都难！       2\n",
       "154207  自作聪明的人，不在少数，尤其现在社会中。自作聪明，反被聪明误；自以为是，必自作自受。不以为然...       0\n",
       "292698                                 悼念930事件。各位一路走好。。。。       2\n",
       "210638                     中国人为什么自愿做房奴？难道租房比一辈子还贷更让人无法接受？       1\n",
       "229423  强烈要求真相！！！。是哦～闹鬼也不太现实，我倒宁可相信这个是真相连续死亡的10位工人不是自杀...       1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw_words</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>282418</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154207</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292698</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210638</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229423</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284993</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210155</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213389</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353829</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102019</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102005</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78043</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168558</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66813</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224341</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301374</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51411</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78725</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62131</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128368</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355014</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10846</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175407</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313093</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349416</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236674</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320089</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81896</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160402</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47136</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203965</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80025</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229482</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190752</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124792</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339353</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335435</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330852</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156239</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115763</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267670</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12268</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267532</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344529</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281431</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249578</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185116</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147248</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316242</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11172</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185771</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148096</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321866</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38309</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301532</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317548</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59733</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20078</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97807</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11453</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        raw_words  target\n",
       "282418      False   False\n",
       "154207      False   False\n",
       "292698      False   False\n",
       "210638      False   False\n",
       "229423      False   False\n",
       "284993      False   False\n",
       "210155      False   False\n",
       "213389      False   False\n",
       "353829      False   False\n",
       "102019      False   False\n",
       "102005      False   False\n",
       "78043       False   False\n",
       "168558      False   False\n",
       "66813       False   False\n",
       "224341      False   False\n",
       "301374      False   False\n",
       "51411       False   False\n",
       "78725       False   False\n",
       "62131       False   False\n",
       "128368      False   False\n",
       "355014      False   False\n",
       "10846       False   False\n",
       "175407      False   False\n",
       "313093      False   False\n",
       "349416      False   False\n",
       "236674      False   False\n",
       "320089      False   False\n",
       "81896       False   False\n",
       "160402      False   False\n",
       "47136       False   False\n",
       "...           ...     ...\n",
       "203965      False   False\n",
       "80025       False   False\n",
       "229482      False   False\n",
       "190752      False   False\n",
       "124792      False   False\n",
       "339353      False   False\n",
       "335435      False   False\n",
       "330852      False   False\n",
       "156239      False   False\n",
       "115763      False   False\n",
       "267670      False   False\n",
       "12268       False   False\n",
       "267532      False   False\n",
       "344529      False   False\n",
       "281431      False   False\n",
       "249578      False   False\n",
       "185116      False   False\n",
       "147248      False   False\n",
       "316242      False   False\n",
       "11172       False   False\n",
       "185771      False   False\n",
       "148096      False   False\n",
       "321866      False   False\n",
       "38309       False   False\n",
       "301532      False   False\n",
       "317548      False   False\n",
       "59733       False   False\n",
       "20078       False   False\n",
       "97807       False   False\n",
       "11453       False   False\n",
       "\n",
       "[8000 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all1.isnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data,test_data,train_target,test_target=train_test_split(df_all1['raw_words']\n",
    "                                                               ,df_all1['target']\n",
    "                                                              ,test_size=0.2\n",
    "                                                              ,random_state=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all1=pd.DataFrame({\"raw_words\":train_data,\"target\":train_target})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_all1.to_csv('df_train.txt',sep='\\t', index=False,header=None,encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw_words</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>279383</th>\n",
       "      <td>今年怎么啦？玉树地震还没消停，吐鲁番又遭大风袭击城郊及农村风力达十级以上，风口风力达十二级以...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142504</th>\n",
       "      <td>真正的佛教徒无需要这样搞试验。他这还是分别心在做怪，根本与修行走有两条道上。圣雄甘地一向以苦...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313399</th>\n",
       "      <td>这个。。。建议港台翻唱制作新时事节目《富士康下》（PS： 现在有音频版不？）</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100595</th>\n",
       "      <td>。醒醒，都快醒醒，我来了。</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327006</th>\n",
       "      <td>回复凤姐估计马上冲出天朝，飞向奥巴马了不是接不接受她的问题，只是想想这样天天做耍猴子的样子在...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                raw_words  target\n",
       "279383  今年怎么啦？玉树地震还没消停，吐鲁番又遭大风袭击城郊及农村风力达十级以上，风口风力达十二级以...       2\n",
       "142504  真正的佛教徒无需要这样搞试验。他这还是分别心在做怪，根本与修行走有两条道上。圣雄甘地一向以苦...       0\n",
       "313399             这个。。。建议港台翻唱制作新时事节目《富士康下》（PS： 现在有音频版不？）       3\n",
       "100595                                      。醒醒，都快醒醒，我来了。       0\n",
       "327006  回复凤姐估计马上冲出天朝，飞向奥巴马了不是接不接受她的问题，只是想想这样天天做耍猴子的样子在...       3"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw_words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23505</th>\n",
       "      <td>那么。。来吧，依靠我。悲催的两个字：爱我。未婚妻～～～我知道的。。。。图片打开后，按Esc ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66795</th>\n",
       "      <td>哈哈。。她就会说，恩，味道非常好。。上面的童鞋说得太好啦！！！伊过假，张到块死父型，二句话够...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194618</th>\n",
       "      <td>匡威联姻SAK 设计夏季低筒手工皮鞋活动告知快来参加银泰注册送礼活动吧！循环抽奖，快把ipo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210243</th>\n",
       "      <td>我都想要耶...HELLO KITTTY 海報 實 物曝光！圖 片中的女孩是超模身型的！明天...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336306</th>\n",
       "      <td>这是家暴现场么囧 为什么每次都是这只被欺负……阿意痛并快乐着啊你哋 再咁 可爱我唔知点算好喇...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                raw_words\n",
       "23505   那么。。来吧，依靠我。悲催的两个字：爱我。未婚妻～～～我知道的。。。。图片打开后，按Esc ...\n",
       "66795   哈哈。。她就会说，恩，味道非常好。。上面的童鞋说得太好啦！！！伊过假，张到块死父型，二句话够...\n",
       "194618  匡威联姻SAK 设计夏季低筒手工皮鞋活动告知快来参加银泰注册送礼活动吧！循环抽奖，快把ipo...\n",
       "210243  我都想要耶...HELLO KITTTY 海報 實 物曝光！圖 片中的女孩是超模身型的！明天...\n",
       "336306  这是家暴现场么囧 为什么每次都是这只被欺负……阿意痛并快乐着啊你哋 再咁 可爱我唔知点算好喇..."
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# testlist=df_test['label']\n",
    "# # trainlist=int(trainlist)\n",
    "# testlist=[int(i) for i in testlist]\n",
    "# ,'target':df_test['label']\n",
    "df_test1=pd.DataFrame({'raw_words':test_data})\n",
    "df_test1.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_test1.to_csv('df_test.txt',sep='\\t', index=False,header=None,encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据导入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#导入Pytorch包\n",
    "\n",
    "import torch\n",
    "\n",
    "import torch.nn as nn\n",
    "\n",
    "from fastNLP.io.loader import CSVLoader\n",
    "\n",
    "dataset_loader = CSVLoader(headers=('raw_words','target'), sep='\\t')\n",
    "testset_loader = CSVLoader( headers=['raw_words'],sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 表示将CSV文件中每一行的第一项将填入'raw_words' field，第二项填入'target' field。\n",
    "\n",
    "# 其中项之间由'\\t'分割开来\n",
    "\n",
    "train_path=r'df_train.txt'\n",
    "\n",
    "test_path=r'df_test.txt'\n",
    "\n",
    "dataset = dataset_loader._load(train_path)\n",
    "\n",
    "testset = testset_loader._load(test_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.39\n"
     ]
    }
   ],
   "source": [
    "# 将句子分成单词形式, 详见DataSet.apply()方法\n",
    "\n",
    "import jieba\n",
    "\n",
    "from itertools import chain\n",
    "\n",
    "print(jieba.__version__)\n",
    "# from itertools import chain\n",
    "\n",
    "#     '''\n",
    "\n",
    "#     @params:\n",
    "\n",
    "#         data: 数据的列表，列表中的每个元素为 [文本字符串，0/1标签] 二元组\n",
    "\n",
    "#     @return: 切分词后的文本的列表，列表中的每个元素为切分后的词序列\n",
    "\n",
    "#     '''\n",
    "\n",
    "def get_tokenized(data,words=True):\n",
    "    def tokenizer(text):\n",
    "        return [tok for tok in jieba.cut(text, cut_all=False)]\n",
    "    if words:\n",
    "\n",
    "        #按词语进行编码\n",
    "\n",
    "        return tokenizer(data)\n",
    "\n",
    "    else:\n",
    "\n",
    "        #按字进行编码\n",
    "\n",
    "        return [tokenizer(review) for review in data]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------------------+--------+\n",
      "| raw_words                           | target |\n",
      "+-------------------------------------+--------+\n",
      "| 今年怎么啦？玉树地震还没消停，吐... | 2      |\n",
      "| 真正的佛教徒无需要这样搞试验。他... | 0      |\n",
      "| 这个。。。建议港台翻唱制作新时事... | 3      |\n",
      "| 。醒醒，都快醒醒，我来了。          | 0      |\n",
      "| 回复凤姐估计马上冲出天朝，飞向奥... | 3      |\n",
      "| 抽奖抽奖抽奖好好关注，天天向上爱... | 0      |\n",
      "| 仁慈的主啊请快点让我找到新的办公... | 3      |\n",
      "| 黄俊英后仲有何人?只有后继无人~ ...  | 0      |\n",
      "| 『热片段』只有短短滴15秒，超搞笑... | 0      |\n",
      "| 。李宇春翻唱合辑                    | 0      |\n",
      "| 是的，小时愿大家心情都好。一起喝... | 0      |\n",
      "| 我国为什么会有那么多的地质灾难?...  | 3      |\n",
      "| 回复要不咱们上Q 聊？？？安逸安逸... | 0      |\n",
      "| 这位陪伴者全程曝光这是干什么？尚... | 0      |\n",
      "| mark 下，一定去望京一号的水煮鱼...  | 0      |\n",
      "| 我也想你们哦~ ，我想你了~ 快快回... | 0      |\n",
      "| 明天一大早要送妹妹去郑州上学了，... | 0      |\n",
      "| 第1000条围脖除了发你还能给谁呢      | 0      |\n",
      "| 等你点评巴黎的炸酱面呢。            | 0      |\n",
      "| 相处时，男人越爱女人，越离失去她... | 2      |\n",
      "| 牛仔裤保养小贴士哦！如何正確 保...  | 0      |\n",
      "| 海绵宝宝，汗~ 我变成公主，开心。... | 2      |\n",
      "| 我怕热，我怕死……福州、杭州、重...   | 2      |\n",
      "| 激动死，就是香港人常说的好high ...  | 1      |\n",
      "| 容易满足，才能快乐！【知足常乐】... | 0      |\n",
      "| 七大美人继《鹿鼎记》戏水图之后的... | 0      |\n",
      "| 。我觉得某些人年纪大了记性就是不... | 2      |\n",
      "| 准行不行啊？难道俺在麻将方面有潜... | 0      |\n",
      "| 网速还能再慢点啊啊啊啊啊top mod...  | 2      |\n",
      "| 一聊天工具管的也太宽了吧！竟然扫... | 1      |\n",
      "| 好久沒 有這 樣 放空,風 吹著臉 ,...  | 0      |\n",
      "| ...                                 | ...    |\n",
      "+-------------------------------------+--------+\n"
     ]
    }
   ],
   "source": [
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 1.577 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------------------+--------+---------------------------+\n",
      "| raw_words                 | target | words                     |\n",
      "+---------------------------+--------+---------------------------+\n",
      "| 今年怎么啦？玉树地震还... | 2      | ['今年', '怎么', '啦'...  |\n",
      "| 真正的佛教徒无需要这样... | 0      | ['真正', '的', '佛教徒... |\n",
      "| 这个。。。建议港台翻唱... | 3      | ['这个', '。', '。', ...  |\n",
      "| 。醒醒，都快醒醒，我来... | 0      | ['。', '醒醒', '，', ...  |\n",
      "| 回复凤姐估计马上冲出天... | 3      | ['回复', '凤姐', '估计... |\n",
      "| 抽奖抽奖抽奖好好关注，... | 0      | ['抽奖', '抽奖', '抽奖... |\n",
      "| 仁慈的主啊请快点让我找... | 3      | ['仁慈', '的', '主', ...  |\n",
      "| 黄俊英后仲有何人?只有...  | 0      | ['黄俊英', '后仲有', ...  |\n",
      "| 『热片段』只有短短滴1...  | 0      | ['『', '热', '片段', ...  |\n",
      "| 。李宇春翻唱合辑          | 0      | ['。', '李宇春', '翻唱... |\n",
      "| 是的，小时愿大家心情都... | 0      | ['是', '的', '，', '小... |\n",
      "| 我国为什么会有那么多的... | 3      | ['我国', '为什么', '会... |\n",
      "| 回复要不咱们上Q 聊？？... | 0      | ['回复', '要', '不', ...  |\n",
      "| 这位陪伴者全程曝光这是... | 0      | ['这位', '陪伴', '者'...  |\n",
      "| mark 下，一定去望京一...  | 0      | ['mark', ' ', '下', '...  |\n",
      "| 我也想你们哦~ ，我想你... | 0      | ['我', '也', '想', '你... |\n",
      "| 明天一大早要送妹妹去郑... | 0      | ['明天', '一大早', '要... |\n",
      "| 第1000条围脖除了发你还... | 0      | ['第', '1000', '条', ...  |\n",
      "| 等你点评巴黎的炸酱面呢... | 0      | ['等', '你', '点评', ...  |\n",
      "| 相处时，男人越爱女人，... | 2      | ['相处', '时', '，', ...  |\n",
      "| 牛仔裤保养小贴士哦！如... | 0      | ['牛仔裤', '保养', '小... |\n",
      "| 海绵宝宝，汗~ 我变成公... | 2      | ['海绵', '宝宝', '，'...  |\n",
      "| 我怕热，我怕死……福州...   | 2      | ['我', '怕热', '，', ...  |\n",
      "| 激动死，就是香港人常说... | 1      | ['激动', '死', '，', ...  |\n",
      "| 容易满足，才能快乐！【... | 0      | ['容易', '满足', '，'...  |\n",
      "| 七大美人继《鹿鼎记》戏... | 0      | ['七大美人继', '《', ...  |\n",
      "| 。我觉得某些人年纪大了... | 2      | ['。', '我', '觉得', ...  |\n",
      "| 准行不行啊？难道俺在麻... | 0      | ['准', '行不行', '啊'...  |\n",
      "| 网速还能再慢点啊啊啊啊... | 2      | ['网速', '还', '能', ...  |\n",
      "| 一聊天工具管的也太宽了... | 1      | ['一', '聊天工具', '管... |\n",
      "| 好久沒 有這 樣 放空,風... | 0      | ['好久', '沒', ' ', '...  |\n",
      "| ...                       | ...    | ...                       |\n",
      "+---------------------------+--------+---------------------------+\n"
     ]
    }
   ],
   "source": [
    "dataset.apply(lambda ins:get_tokenized(ins['raw_words']), new_field_name='words', is_input=True)\n",
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------+--------+----------------------+---------+\n",
      "| raw_words            | target | words                | seq_len |\n",
      "+----------------------+--------+----------------------+---------+\n",
      "| 今年怎么啦？玉树...  | 2      | ['今年', '怎么',...  | 54      |\n",
      "| 真正的佛教徒无需...  | 0      | ['真正', '的', '...  | 108     |\n",
      "| 这个。。。建议港...  | 3      | ['这个', '。', '...  | 26      |\n",
      "| 。醒醒，都快醒醒...  | 0      | ['。', '醒醒', '...  | 10      |\n",
      "| 回复凤姐估计马上...  | 3      | ['回复', '凤姐',...  | 68      |\n",
      "| 抽奖抽奖抽奖好好...  | 0      | ['抽奖', '抽奖',...  | 67      |\n",
      "| 仁慈的主啊请快点...  | 3      | ['仁慈', '的', '...  | 22      |\n",
      "| 黄俊英后仲有何人...  | 0      | ['黄俊英', '后仲...  | 69      |\n",
      "| 『热片段』只有短...  | 0      | ['『', '热', '片...  | 60      |\n",
      "| 。李宇春翻唱合辑...  | 0      | ['。', '李宇春',...  | 4       |\n",
      "| 是的，小时愿大家...  | 0      | ['是', '的', '，...  | 94      |\n",
      "| 我国为什么会有那...  | 3      | ['我国', '为什么...  | 49      |\n",
      "| 回复要不咱们上Q ...  | 0      | ['回复', '要', '...  | 36      |\n",
      "| 这位陪伴者全程曝...  | 0      | ['这位', '陪伴',...  | 9       |\n",
      "| mark 下，一定去望... | 0      | ['mark', ' ', '下... | 93      |\n",
      "| 我也想你们哦~ ，...  | 0      | ['我', '也', '想...  | 17      |\n",
      "| 明天一大早要送妹...  | 0      | ['明天', '一大早...  | 19      |\n",
      "| 第1000条围脖除了...  | 0      | ['第', '1000', '...  | 12      |\n",
      "| 等你点评巴黎的炸...  | 0      | ['等', '你', '点...  | 8       |\n",
      "| 相处时，男人越爱...  | 2      | ['相处', '时', '...  | 88      |\n",
      "| 牛仔裤保养小贴士...  | 0      | ['牛仔裤', '保养...  | 14      |\n",
      "| 海绵宝宝，汗~ 我...  | 2      | ['海绵', '宝宝',...  | 43      |\n",
      "| 我怕热，我怕死…...   | 2      | ['我', '怕热', '...  | 19      |\n",
      "| 激动死，就是香港...  | 1      | ['激动', '死', '...  | 53      |\n",
      "| 容易满足，才能快...  | 0      | ['容易', '满足',...  | 9       |\n",
      "| 七大美人继《鹿鼎...  | 0      | ['七大美人继', '...  | 14      |\n",
      "| 。我觉得某些人年...  | 2      | ['。', '我', '觉...  | 43      |\n",
      "| 准行不行啊？难道...  | 0      | ['准', '行不行',...  | 15      |\n",
      "| 网速还能再慢点啊...  | 2      | ['网速', '还', '...  | 14      |\n",
      "| 一聊天工具管的也...  | 1      | ['一', '聊天工具...  | 95      |\n",
      "| 好久沒 有這 樣 放... | 0      | ['好久', '沒', '...  | 37      |\n",
      "| ...                  | ...    | ...                  | ...     |\n",
      "+----------------------+--------+----------------------+---------+\n"
     ]
    }
   ],
   "source": [
    "dataset.apply(lambda ins: len(ins['words']) ,new_field_name='seq_len', is_input=True)\n",
    "\n",
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------+--------+----------------------+---------+\n",
      "| raw_words            | target | words                | seq_len |\n",
      "+----------------------+--------+----------------------+---------+\n",
      "| 今年怎么啦？玉树...  | 2      | ['今年', '怎么',...  | 54      |\n",
      "| 真正的佛教徒无需...  | 0      | ['真正', '的', '...  | 108     |\n",
      "| 这个。。。建议港...  | 3      | ['这个', '。', '...  | 26      |\n",
      "| 。醒醒，都快醒醒...  | 0      | ['。', '醒醒', '...  | 10      |\n",
      "| 回复凤姐估计马上...  | 3      | ['回复', '凤姐',...  | 68      |\n",
      "| 抽奖抽奖抽奖好好...  | 0      | ['抽奖', '抽奖',...  | 67      |\n",
      "| 仁慈的主啊请快点...  | 3      | ['仁慈', '的', '...  | 22      |\n",
      "| 黄俊英后仲有何人...  | 0      | ['黄俊英', '后仲...  | 69      |\n",
      "| 『热片段』只有短...  | 0      | ['『', '热', '片...  | 60      |\n",
      "| 。李宇春翻唱合辑...  | 0      | ['。', '李宇春',...  | 4       |\n",
      "| 是的，小时愿大家...  | 0      | ['是', '的', '，...  | 94      |\n",
      "| 我国为什么会有那...  | 3      | ['我国', '为什么...  | 49      |\n",
      "| 回复要不咱们上Q ...  | 0      | ['回复', '要', '...  | 36      |\n",
      "| 这位陪伴者全程曝...  | 0      | ['这位', '陪伴',...  | 9       |\n",
      "| mark 下，一定去望... | 0      | ['mark', ' ', '下... | 93      |\n",
      "| 我也想你们哦~ ，...  | 0      | ['我', '也', '想...  | 17      |\n",
      "| 明天一大早要送妹...  | 0      | ['明天', '一大早...  | 19      |\n",
      "| 第1000条围脖除了...  | 0      | ['第', '1000', '...  | 12      |\n",
      "| 等你点评巴黎的炸...  | 0      | ['等', '你', '点...  | 8       |\n",
      "| 相处时，男人越爱...  | 2      | ['相处', '时', '...  | 88      |\n",
      "| 牛仔裤保养小贴士...  | 0      | ['牛仔裤', '保养...  | 14      |\n",
      "| 海绵宝宝，汗~ 我...  | 2      | ['海绵', '宝宝',...  | 43      |\n",
      "| 我怕热，我怕死…...   | 2      | ['我', '怕热', '...  | 19      |\n",
      "| 激动死，就是香港...  | 1      | ['激动', '死', '...  | 53      |\n",
      "| 容易满足，才能快...  | 0      | ['容易', '满足',...  | 9       |\n",
      "| 七大美人继《鹿鼎...  | 0      | ['七大美人继', '...  | 14      |\n",
      "| 。我觉得某些人年...  | 2      | ['。', '我', '觉...  | 43      |\n",
      "| 准行不行啊？难道...  | 0      | ['准', '行不行',...  | 15      |\n",
      "| 网速还能再慢点啊...  | 2      | ['网速', '还', '...  | 14      |\n",
      "| 一聊天工具管的也...  | 1      | ['一', '聊天工具...  | 95      |\n",
      "| 好久沒 有這 樣 放... | 0      | ['好久', '沒', '...  | 37      |\n",
      "| ...                  | ...    | ...                  | ...     |\n",
      "+----------------------+--------+----------------------+---------+\n"
     ]
    }
   ],
   "source": [
    "dataset.apply(lambda x: int(x['target']), new_field_name='target', is_target=True)\n",
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------------------+---------------------------+---------+\n",
      "| raw_words                 | words                     | seq_len |\n",
      "+---------------------------+---------------------------+---------+\n",
      "| 那么。。来吧，依靠我。... | ['那么', '。', '。', ...  | 48      |\n",
      "| 哈哈。。她就会说，恩，... | ['哈哈', '。', '。', ...  | 132     |\n",
      "| 匡威联姻SAK 设计夏季低... | ['匡威', '联姻', 'SAK...  | 41      |\n",
      "| 我都想要耶...HELLO KI...  | ['我', '都', '想要', ...  | 39      |\n",
      "| 这是家暴现场么囧 为什...  | ['这是', '家暴', '现场... | 57      |\n",
      "| 来时空去时匆，生死仿佛... | ['来', '时空', '去', ...  | 47      |\n",
      "| 你偷偷传给我，我不告诉... | ['你', '偷偷', '传给'...  | 92      |\n",
      "| 哈哈···冲动狂~ 每个星...  | ['哈哈', '·', '·', '·...  | 35      |\n",
      "| 三分法－－我们使用三分... | ['三分法', '－', '－'...  | 23      |\n",
      "| 不能看！看了美食就忘不... | ['不能', '看', '！', ...  | 19      |\n",
      "| 北京特产喝酸奶~ ~         | ['北京', '特产', '喝'...  | 7       |\n",
      "| 你一看到我，气质就变得... | ['你', '一', '看到', ...  | 39      |\n",
      "| 看看哩个版本正啊！狼閚... | ['看看', '哩个', '版本... | 30      |\n",
      "| 作为典型的80后，唐山地... | ['作为', '典型', '的'...  | 73      |\n",
      "| : 2010年的夏天很是特别... | [':', ' ', '2010', '年... | 75      |\n",
      "| 新版范进中举。不遵医嘱... | ['新版', '范进中', '举... | 12      |\n",
      "| 帅气面对这样的口罩，您... | ['帅气', '面对', '这样... | 16      |\n",
      "| 老子说，慎终若始，则无... | ['老子', '说', '，', ...  | 25      |\n",
      "| 办届亚运，雷人事件枚不... | ['办届', '亚运', '，'...  | 70      |\n",
      "| 哈哈~ 喵了个咪·三角什...  | ['哈哈', '~', ' ', '喵... | 12      |\n",
      "| 開 車 慢點 咯~ 我乃新...  | ['開', ' ', '車', ' '...  | 20      |\n",
      "| ★近来在80后小夫妻中很...  | ['★', '近来', '在', ...   | 62      |\n",
      "| 看来今天还是要一颗星线... | ['看来', '今天', '还是... | 16      |\n",
      "| 经过一段纠结的时光，我... | ['经过', '一段', '纠结... | 15      |\n",
      "| 突然好有冲动想..........  | ['突然', '好', '有', ...  | 21      |\n",
      "| 下次试试。一張 貼 紙 ...  | ['下次', '试试', '。'...  | 17      |\n",
      "| 啊哈垮棚啦，要命啦！！... | ['啊哈', '垮', '棚', ...  | 32      |\n",
      "| 自微博日报()菲警与劫匪... | ['自微博', '日报', '(...  | 49      |\n",
      "| 又恶心又想看是一种什么... | ['又', '恶心', '又', ...  | 38      |\n",
      "| 我靠，居然看到象牙啦！... | ['我', '靠', '，', '居... | 53      |\n",
      "| 我倒数的双子的人想太多... | ['我', '倒数', '的', ...  | 90      |\n",
      "| ...                       | ...                       | ...     |\n",
      "+---------------------------+---------------------------+---------+\n"
     ]
    }
   ],
   "source": [
    "#testset.apply(lambda ins: list(chain.from_iterable(get_tokenized(ins['raw_words']))), new_field_name='words', is_input=True)\n",
    "\n",
    "testset.apply(lambda ins: get_tokenized(ins['raw_words']), new_field_name='words', is_input=True)\n",
    "\n",
    "testset.apply(lambda ins: len(ins['words']) ,new_field_name='seq_len',is_input=True)\n",
    "print(testset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###\n",
    "\n",
    "from fastNLP import Vocabulary\n",
    "\n",
    "#将DataSet按照ratio的比例拆分，返回两个DataSet\n",
    "\n",
    "#ratio (float) -- 0<ratio<1, 返回的第一个DataSet拥有 (1-ratio) 这么多数据，第二个DataSet拥有`ratio`这么多数据\n",
    "\n",
    "train_data, dev_data = dataset.split(0.1, shuffle=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------+--------+---------------------+---------+\n",
      "| raw_words            | target | words               | seq_len |\n",
      "+----------------------+--------+---------------------+---------+\n",
      "| 快乐就好~ 我就這...  | 0      | ['快乐', '就', '... | 17      |\n",
      "| 关注别人，就是关...  | 0      | ['关注', '别人',... | 34      |\n",
      "| ：克来登大学转世...  | 0      | ['：', '克来', '... | 59      |\n",
      "| 回复江老大被我戳...  | 0      | ['回复', '江', '... | 34      |\n",
      "| 80度天蝎1000度牛...  | 0      | ['80', '度', '天... | 47      |\n",
      "| 祈祷有用的话，还...  | 2      | ['祈祷', '有用',... | 54      |\n",
      "| 瞄呜，瞄呜，还是...  | 1      | ['瞄', '呜', '，... | 11      |\n",
      "| 回覆早长久的温暖...  | 0      | ['回覆', '早', '... | 50      |\n",
      "| 阿木童在殡仪馆抓...  | 0      | ['阿木童', '在',... | 9       |\n",
      "| 美人心计大爱刘章...  | 0      | ['美人', '心计',... | 6       |\n",
      "| 我倒觉得老先生特...  | 0      | ['我', '倒', '觉... | 44      |\n",
      "| 昨天接到杭州萧山...  | 1      | ['昨天', '接到',... | 50      |\n",
      "| 想起了陪伴我的那...  | 0      | ['想起', '了', '... | 12      |\n",
      "| 哎~ 装什么装啊姑...  | 0      | ['哎', '~', ' ',... | 100     |\n",
      "| 一定要好好学习~ ...  | 0      | ['一定', '要', '... | 50      |\n",
      "| 明天清早出门，「...  | 0      | ['明天', '清早',... | 21      |\n",
      "| 轮胎人据传是米其...  | 0      | ['轮胎', '人', '... | 7       |\n",
      "| 。我居然看完了口...  | 3      | ['。', '我', '居... | 9       |\n",
      "| 没人陪我看恐怖片...  | 2      | ['没人', '陪', '... | 17      |\n",
      "| 刚才同事小郑跟我...  | 2      | ['刚才', '同事',... | 68      |\n",
      "| 我想睇盗梦…我想...   | 2      | ['我', '想', '睇... | 19      |\n",
      "| 这个笑话原来有听...  | 0      | ['这个', '笑话',... | 94      |\n",
      "| 高清剧照发布！Yo...  | 0      | ['高清', '剧照',... | 52      |\n",
      "| 昨日停水，忘关水...  | 2      | ['昨日', '停水',... | 39      |\n",
      "| 加油~ 俞灏明灏明...  | 0      | ['加油', '~', ' ... | 51      |\n",
      "| 【铁罐揭秘】腾讯...  | 0      | ['【', '铁罐', '... | 23      |\n",
      "| 我财运不咋地啊.还... | 0      | ['我', '财运', '... | 123     |\n",
      "| 老婆~ 无法坦诚相...  | 2      | ['老婆', '~', ' ... | 19      |\n",
      "| 东风汽车有限公司...  | 3      | ['东风汽车', '有... | 68      |\n",
      "| 唉，算了，没有李...  | 1      | ['唉', '，', '算... | 113     |\n",
      "| 祝福！今天是全国...  | 2      | ['祝福', '！', '... | 77      |\n",
      "| ...                  | ...    | ...                 | ...     |\n",
      "+----------------------+--------+---------------------+---------+\n"
     ]
    }
   ],
   "source": [
    "print(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5760 640 1600\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data),len(dev_data),len(testset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Vocabulary(['今年', '怎么', '啦', '？', '玉树']...)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = Vocabulary(min_freq=2).from_dataset(dataset, field_name='words')\n",
    "\n",
    "vocab.index_dataset(train_data, dev_data, testset, field_name='words', new_field_name='words')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1996 out of 15021 words in the pre-training embedding.\n"
     ]
    }
   ],
   "source": [
    "from fastNLP.embeddings import StaticEmbedding,StackEmbedding\n",
    "\n",
    "fastnlp_embed = StaticEmbedding(vocab, model_dir_or_name='cn-char-fastnlp-100d',min_freq=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNNText(\n",
      "  (embed): Embedding(\n",
      "    (embed): StaticEmbedding(\n",
      "      (dropout_layer): Dropout(p=0)\n",
      "      (embedding): Embedding(15021, 100, padding_idx=0)\n",
      "    )\n",
      "    (dropout): Dropout(p=0.0)\n",
      "  )\n",
      "  (conv_pool): ConvMaxpool(\n",
      "    (convs): ModuleList(\n",
      "      (0): Conv1d(100, 30, kernel_size=(1,), stride=(1,), bias=False)\n",
      "      (1): Conv1d(100, 40, kernel_size=(3,), stride=(1,), padding=(1,), bias=False)\n",
      "      (2): Conv1d(100, 50, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)\n",
      "    )\n",
      "  )\n",
      "  (dropout): Dropout(p=0.1)\n",
      "  (fc): Linear(in_features=120, out_features=4, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from fastNLP.models import CNNText\n",
    "\n",
    "model_CNN = CNNText(fastnlp_embed, num_classes=4,dropout=0.1)\n",
    "\n",
    "print(model_CNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input fields after batch(if batch size is 2):\n",
      "\twords: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 34]) \n",
      "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
      "target fields after batch(if batch size is 2):\n",
      "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n",
      "\n",
      "training epochs started 2020-06-18-17-55-10-654843\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=1800.0), HTML(value='')), layout=Layout(d…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 0.96 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 1/10. Step:180/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.551562\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 0.99 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 2/10. Step:360/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.570312\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 0.97 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 3/10. Step:540/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.523437\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 0.97 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 4/10. Step:720/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.529687\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 0.89 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 5/10. Step:900/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.503125\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 1.54 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 6/10. Step:1080/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.5375\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 1.41 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 7/10. Step:1260/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.5\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 1.22 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 8/10. Step:1440/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.521875\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 1.83 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 9/10. Step:1620/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.532812\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=20.0), HTML(value='')), layout=Layout(dis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Evaluate data in 2.11 seconds!\n",
      "\r",
      "Evaluation on dev at Epoch 10/10. Step:1800/1800: \n",
      "\r",
      "AccuracyMetric: acc=0.526562\n",
      "\n",
      "\r",
      "Reloaded the best model.\n",
      "\n",
      "In Epoch:2/Step:360, got best dev performance:\n",
      "AccuracyMetric: acc=0.570312\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'best_epoch': 2,\n",
       " 'best_eval': {'AccuracyMetric': {'acc': 0.570312}},\n",
       " 'best_step': 360,\n",
       " 'seconds': 323.16}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from fastNLP import Trainer, CrossEntropyLoss, AccuracyMetric,BCELoss\n",
    "\n",
    "trainer_CNN = Trainer(model=model_CNN, train_data=train_data, dev_data=dev_data,loss=CrossEntropyLoss(), metrics=AccuracyMetric())\n",
    "\n",
    "trainer_CNN.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "demo=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "\n",
    "def batch_predict(model,data):\n",
    "#     submission=pd.DataFrame(columns=['Prediction'])\n",
    "#    submission = pd.DataFrame(columns=['ID','Prediction'])\n",
    "    demo1=[]\n",
    "    for i in range(len(data)):\n",
    "    #for i in range(5):\n",
    "#         print(data.words[i])\n",
    "        tensor = torch.tensor(data.words[i])\n",
    "        pred = model.predict(tensor.view(1,-1))\n",
    "#         print(pred)\n",
    "        prob = pred['pred'].numpy()[0]\n",
    "#         print(\"pred:%.2f\"%(prob))\n",
    "#         print('='*50)\n",
    "#         print(type(prob))\n",
    "#         s2 = pd.Series([float(prob)], index=['Prediction'])\n",
    "        demo1.append(prob)\n",
    "#         print(prob)\n",
    "# #         print(s2)\n",
    "#         submission = submission.append(s2, ignore_index=True)\n",
    "#         submission['Prediction'] = submission.Prediction .astype(int)\n",
    "#         submission['']\n",
    "#         submission['Prediction'] = submission.Prediction.astype(float) \n",
    "    return demo1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "demo=batch_predict(model_CNN,testset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 1, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 1, 0, 3, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 3, 0, 3, 0, 0, 0, 1, 0, 0, 3, 1, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "print(demo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1600\n"
     ]
    }
   ],
   "source": [
    "print(len(demo))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.561875\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "print(accuracy_score(test_target,demo))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'fastNLP.core.trainer.Trainer'>\n"
     ]
    }
   ],
   "source": [
    "print(type(trainer_CNN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}