bert-imdb.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-03T09:19:08.328308Z","iopub.execute_input":"2021-08-03T09:19:08.328728Z","iopub.status.idle":"2021-08-03T09:19:08.338546Z","shell.execute_reply.started":"2021-08-03T09:19:08.328629Z","shell.execute_reply":"2021-08-03T09:19:08.337436Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\n%cd / ","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:19:08.340514Z","iopub.execute_input":"2021-08-03T09:19:08.341228Z","iopub.status.idle":"2021-08-03T09:19:08.803627Z","shell.execute_reply.started":"2021-08-03T09:19:08.341139Z","shell.execute_reply":"2021-08-03T09:19:08.802465Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"/\n","output_type":"stream"}]},{"cell_type":"code","source":"def load_dataset(paths):\n    data = []\n    label = []\n    value = 0\n    for path in paths:\n        print(path)\n        if path.split('/')[-1] == 'pos':\n            value = 1\n        else:\n            value = 0\n        for son in os.listdir(path):\n            new_path = os.path.join(path,son)\n            with open(new_path,'r') as f:\n                lines = f.readlines()\n                for line in lines:\n                    data.append(line)\n                    label.append(value)\n    return data,label\ntrain_data,train_label = load_dataset(['/kaggle/input/acllmdb/aclImdb/train/pos','/kaggle/input/acllmdb/aclImdb/train/neg'])#'/kaggle/input/acllmdb/aclImdb/train/neg'\ntest_data,test_label = load_dataset(['/kaggle/input/acllmdb/aclImdb/test/pos','/kaggle/input/acllmdb/aclImdb/test/neg'])#,'/kaggle/input/acllmdb/aclImdb/test/neg'\n","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:19:08.805512Z","iopub.execute_input":"2021-08-03T09:19:08.805929Z","iopub.status.idle":"2021-08-03T09:20:28.228987Z","shell.execute_reply.started":"2021-08-03T09:19:08.805884Z","shell.execute_reply":"2021-08-03T09:20:28.227963Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"/kaggle/input/acllmdb/aclImdb/train/pos\n/kaggle/input/acllmdb/aclImdb/train/neg\n/kaggle/input/acllmdb/aclImdb/test/pos\n/kaggle/input/acllmdb/aclImdb/test/neg\n","output_type":"stream"}]},{"cell_type":"code","source":"# !pip install transformers\nfrom transformers import BertTokenizer, BertModel\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\nbertmodel = BertModel.from_pretrained('bert-base-uncased')\n# ret = tokenizer.tokenize('what are you doing now')\n# input_ids = tokenizer.convert_tokens_to_ids(ret)\n# inputs = tokenizer.encode('Hello, my dog is cute',max_length = 3)\n# inputssss = tokenizer('Hello, my dog is cute',max_length = 512)\n# print(inputssss)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:20:28.230466Z","iopub.execute_input":"2021-08-03T09:20:28.230876Z","iopub.status.idle":"2021-08-03T09:20:34.591847Z","shell.execute_reply.started":"2021-08-03T09:20:28.230832Z","shell.execute_reply":"2021-08-03T09:20:34.590829Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight']\n- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","output_type":"stream"}]},{"cell_type":"code","source":"# for name,para in model.named_parameters():\n#     print(name,':',para)\n# print(model.config.hidden_size)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:20:34.593278Z","iopub.execute_input":"2021-08-03T09:20:34.593697Z","iopub.status.idle":"2021-08-03T09:20:34.600815Z","shell.execute_reply.started":"2021-08-03T09:20:34.593664Z","shell.execute_reply":"2021-08-03T09:20:34.596998Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nclass myModel(torch.nn.Module):\n    def __init__(self,bertmodel,dropout=None):\n        super(myModel,self).__init__()\n        self.bertmodel = bertmodel\n        self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)\n        # 语义匹配任务: 相似、不相似 2 分类任务\n        self.classifier = nn.Linear(self.bertmodel.config.hidden_size, 2)\n    def forward(self,input_ids,token_type_ids,attention_mask):\n        outputs= self.bertmodel(input_ids,token_type_ids = token_type_ids, attention_mask=attention_mask)#\n        outputs = outputs.pooler_output\n        outputs = self.dropout(outputs)\n        # 基于文本对的语义表示向量进行 2 分类任务\n        logits = self.classifier(outputs)\n        return logits\nmodel = myModel(bertmodel)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)\nmodel = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:20:34.602502Z","iopub.execute_input":"2021-08-03T09:20:34.603013Z","iopub.status.idle":"2021-08-03T09:20:37.368268Z","shell.execute_reply.started":"2021-08-03T09:20:34.602968Z","shell.execute_reply":"2021-08-03T09:20:37.367422Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"cuda\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import numpy as np\n# def convert_examples_to_feature(examples,tokenizer =tokenizer,max_length = 512):\n#     data = []\n#     for text in examples:\n#         encoder_inputs = tokenizer(text,max_length)\n#         input_ids = encoder_inputs['input_ids']\n#         data.append(input_ids)\n#     return data\n\n# train_data = convert_examples_to_feature(train_data)\n# test_data = convert_examples_to_feature(test_data)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:20:37.369612Z","iopub.execute_input":"2021-08-03T09:20:37.369977Z","iopub.status.idle":"2021-08-03T09:20:37.373821Z","shell.execute_reply.started":"2021-08-03T09:20:37.369940Z","shell.execute_reply":"2021-08-03T09:20:37.372890Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"# print(len(train_data))\n# print(train_data[0])","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:20:37.377822Z","iopub.execute_input":"2021-08-03T09:20:37.378209Z","iopub.status.idle":"2021-08-03T09:20:37.387971Z","shell.execute_reply.started":"2021-08-03T09:20:37.378176Z","shell.execute_reply":"2021-08-03T09:20:37.387030Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset\nclass myDataset(Dataset):\n    def __init__(self,data,label):\n        super(myDataset).__init__()\n        self.input_ids = []\n        self.token_type_ids = []\n        self.attention_mask = []\n        self.label = label\n        for text in data:\n            encoder_inputs = tokenizer(text,max_length = 128)\n            self.input_ids.append(encoder_inputs['input_ids'])\n            self.token_type_ids.append(encoder_inputs['token_type_ids'])\n            self.attention_mask.append(encoder_inputs['attention_mask'])\n    def __getitem__(self,idx):\n        input_ids = self.input_ids[idx]\n        token_type_ids = self.token_type_ids[idx]\n        attention_mask = self.attention_mask[idx]\n        label = self.label[idx]\n        return torch.tensor(input_ids, dtype=torch.long),torch.tensor(token_type_ids, dtype=torch.long),torch.tensor(attention_mask, dtype=torch.long),torch.tensor(label, dtype=torch.long)\n    def __len__(self):\n        return len(self.input_ids)\ntrain_dataset = myDataset(train_data,train_label)\ntest_dataset = myDataset(test_data,test_label)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:20:37.389857Z","iopub.execute_input":"2021-08-03T09:20:37.390231Z","iopub.status.idle":"2021-08-03T09:26:50.728610Z","shell.execute_reply.started":"2021-08-03T09:20:37.390194Z","shell.execute_reply":"2021-08-03T09:26:50.727752Z"},"trusted":true},"execution_count":9,"outputs":[{"name":"stderr","text":"Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n","output_type":"stream"}]},{"cell_type":"code","source":"import numpy as np\ndef my_collate_fn(data):\n    new_input_ids = []\n    new_token_type_ids = []\n    new_attention_mask = []\n    new_y = []\n    max_length = 0\n    for input_ids,token_type_ids,attention_mask,label in data:\n        token_length = len(input_ids)\n        max_length = max(max_length,token_length)\n    if max_length > 512:\n        max_length = 512\n    for input_ids,token_type_ids,attention_mask,label in data:\n        token_length = len(input_ids)\n        input_ids = np.array(input_ids)\n        token_type_ids = np.array(token_type_ids)\n        attention_mask = np.array(attention_mask)\n#         label = np.array(label)   因为这里取出的label就是一个tensor（1），其里面是一个int整数，如果转成np.array，后面再转tensor就会报no len（）错，错误描述可以看下一个cell\n        if token_length < max_length:\n            input_ids = np.concatenate([input_ids[:-1],[tokenizer.pad_token_id] * (max_length - token_length) + [tokenizer.sep_token_id]],axis = 0)\n            token_type_ids = np.array(token_type_ids)\n            token_type_ids = np.concatenate([token_type_ids,[0] * (max_length - token_length)],axis = 0)\n            attention_mask = np.array(attention_mask)\n            attention_mask = np.concatenate([attention_mask,[0]* (max_length - token_length)],axis = 0)\n        new_input_ids.append(input_ids)\n        new_token_type_ids.append(token_type_ids)\n        new_attention_mask.append(attention_mask)\n        new_y.append(label)\n    new_input_ids = torch.tensor(new_input_ids,dtype=torch.long)\n    new_token_type_ids = torch.tensor(new_token_type_ids,dtype=torch.long)\n    new_attention_mask = torch.tensor(new_attention_mask,dtype=torch.long)\n    new_y = torch.tensor(new_y,dtype=torch.long)\n    return new_input_ids, new_token_type_ids,new_attention_mask,new_y","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:26:50.730084Z","iopub.execute_input":"2021-08-03T09:26:50.730437Z","iopub.status.idle":"2021-08-03T09:26:50.742288Z","shell.execute_reply.started":"2021-08-03T09:26:50.730389Z","shell.execute_reply":"2021-08-03T09:26:50.741348Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"# a = torch.tensor(1)\n# b = torch.tensor(2)\n# c = np.array(a)\n# d = np.array(b)\n# e = [c,d]\n# print(torch.tensor(e))\n# TypeError: len() of unsized object  会报这个错是因为 a中值是一个int型，其没有len（）函数，而在转tensor时，会调取其len（）,故报错，因此上面的label不能转成np.ndarray","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:26:50.743615Z","iopub.execute_input":"2021-08-03T09:26:50.744282Z","iopub.status.idle":"2021-08-03T09:26:50.754197Z","shell.execute_reply.started":"2021-08-03T09:26:50.744244Z","shell.execute_reply":"2021-08-03T09:26:50.753357Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\nprint(len(train_dataset))\ntrain_dataset, valid_dataset = torch.utils.data.random_split(train_dataset, [20000, 5000])\ntrain_data_loader = DataLoader(\n                train_dataset,\n                batch_size = 64,\n                collate_fn = my_collate_fn,\n                shuffle=True\n)\nvalid_data_loader = DataLoader(\n                valid_dataset,\n                batch_size = 64,\n                collate_fn = my_collate_fn,\n                shuffle=True\n)\ntest_data_loader = DataLoader(\n                test_dataset,\n                batch_size = 32,\n                shuffle=True,\n                collate_fn = my_collate_fn\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:26:50.755595Z","iopub.execute_input":"2021-08-03T09:26:50.755961Z","iopub.status.idle":"2021-08-03T09:26:50.770749Z","shell.execute_reply.started":"2021-08-03T09:26:50.755926Z","shell.execute_reply":"2021-08-03T09:26:50.769514Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"25000\n","output_type":"stream"}]},{"cell_type":"code","source":"count = 0 \nfor batch in valid_dataset:\n    if batch[-1] == 0:\n        count += 1\nprint(count / 5000)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:26:50.772311Z","iopub.execute_input":"2021-08-03T09:26:50.772764Z","iopub.status.idle":"2021-08-03T09:26:50.980639Z","shell.execute_reply.started":"2021-08-03T09:26:50.772707Z","shell.execute_reply":"2021-08-03T09:26:50.979604Z"},"trusted":true},"execution_count":13,"outputs":[{"name":"stdout","text":"0.499\n","output_type":"stream"}]},{"cell_type":"code","source":"import time\nimport torch.nn as nn\nfrom torch import optim\nfrom torch.optim import lr_scheduler\ncriterion = nn.CrossEntropyLoss()\ncriterion = criterion.to(device)\noptimizer = optim.Adam(model.parameters(), lr=0.0001)\nscheduler = lr_scheduler.MultiStepLR(optimizer,milestones=[20,80],gamma = 0.9)\nepochs = 5\n# !pip install torchmetrics\nimport torchmetrics as metrics\ntrain_accuracy = metrics.Accuracy()\ntrain_accuracy.to(device)\nckpt_dir = './'","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:26:50.982237Z","iopub.execute_input":"2021-08-03T09:26:50.982868Z","iopub.status.idle":"2021-08-03T09:26:51.582213Z","shell.execute_reply.started":"2021-08-03T09:26:50.982822Z","shell.execute_reply":"2021-08-03T09:26:51.581160Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:26:51.583777Z","iopub.execute_input":"2021-08-03T09:26:51.584211Z","iopub.status.idle":"2021-08-03T09:26:51.588145Z","shell.execute_reply.started":"2021-08-03T09:26:51.584158Z","shell.execute_reply":"2021-08-03T09:26:51.587169Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nglobal_step = 0\ntic_train = time.time()\nmodel.train()\nfor epoch in range(1, epochs + 1):\n    train_loss = 0\n    for step, batch in enumerate(train_data_loader, start=1):\n        optimizer.zero_grad()\n        input_ids,token_type_ids,attention_mask,labels = batch\n        input_ids = input_ids.to(device,dtype=torch.long)\n        token_type_ids = token_type_ids.to(device,dtype=torch.long)\n        attention_mask =attention_mask.to(device,dtype=torch.long)\n        labels = labels.to(device,dtype=torch.long)\n        # 喂数据给model\n        probs = model(input_ids = input_ids, token_type_ids = token_type_ids, attention_mask = attention_mask)\n        # 计算损失函数值\n        loss = criterion(probs, labels)\n        train_loss += loss\n        loss.backward()\n#         for name, parms in model.named_parameters():\n# #             print(parms.data)\n#             print('-->name:', name, '-->grad_requirs:', parms.requires_grad, '--weight', torch.mean(parms.data), ' -->grad_value:', torch.mean(parms.grad))\n        optimizer.step()\n        scheduler.step()\n        # 预测分类概率值\n        # 计算acc\n        batch_acc = train_accuracy(probs, labels)\n        global_step += 1\n        if global_step % 30 == 0:\n#             acc = train_accuracy.compute()\n            print(\n                \"global step %d, epoch: %d, batch: %d, loss: %.5f, accu: %.5f, speed: %.2f step/s\"\n                % (global_step, epoch, step, loss, batch_acc,\n                    10 / (time.time() - tic_train)))\n            tic_train = time.time()\n        \n    print('epoch:%d,total_loss:%.5f,accu:%.5f' %(epoch,train_loss * epoch / global_step, train_accuracy.compute()))\n    \n    # 评估当前训练的模型  \n    save_dir = os.path.join(ckpt_dir, \"model_%d\" % global_step)\n    if not os.path.exists(save_dir):\n        os.makedirs(save_dir) \n    with torch.no_grad():\n        torch.cuda.empty_cache()\n        valid_accuracy = metrics.Accuracy()\n        valid_accuracy.to(device)\n        for  i, batch in enumerate(valid_data_loader):\n            input_ids,token_type_ids,attention_mask,labels = batch\n            input_ids = input_ids.to(device,dtype=torch.long)\n            token_type_ids = token_type_ids.to(device,dtype=torch.long)\n            attention_mask =attention_mask.to(device,dtype=torch.long)\n            labels = labels.to(device,dtype=torch.long)\n            probs = model(input_ids = input_ids, token_type_ids = token_type_ids, attention_mask = attention_mask)\n            valid_accuracy(probs,labels)\n        acc = valid_accuracy.compute()\n        print('eval_acc: %.5f ' % acc)\n        torch.save(model.state_dict(),os.path.join(save_dir,'model.pt'))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:26:51.589797Z","iopub.execute_input":"2021-08-03T09:26:51.590415Z","iopub.status.idle":"2021-08-03T09:47:37.618996Z","shell.execute_reply.started":"2021-08-03T09:26:51.590358Z","shell.execute_reply":"2021-08-03T09:47:37.618094Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"global step 30, epoch: 1, batch: 30, loss: 0.30070, accu: 0.90625, speed: 0.45 step/s\nglobal step 60, epoch: 1, batch: 60, loss: 0.28492, accu: 0.90625, speed: 0.46 step/s\nglobal step 90, epoch: 1, batch: 90, loss: 0.34072, accu: 0.82812, speed: 0.46 step/s\nglobal step 120, epoch: 1, batch: 120, loss: 0.31917, accu: 0.89062, speed: 0.46 step/s\nglobal step 150, epoch: 1, batch: 150, loss: 0.22120, accu: 0.90625, speed: 0.46 step/s\nglobal step 180, epoch: 1, batch: 180, loss: 0.33422, accu: 0.85938, speed: 0.46 step/s\nglobal step 210, epoch: 1, batch: 210, loss: 0.30382, accu: 0.85938, speed: 0.46 step/s\nglobal step 240, epoch: 1, batch: 240, loss: 0.24825, accu: 0.89062, speed: 0.46 step/s\nglobal step 270, epoch: 1, batch: 270, loss: 0.28896, accu: 0.82812, speed: 0.46 step/s\nglobal step 300, epoch: 1, batch: 300, loss: 0.41404, accu: 0.85938, speed: 0.46 step/s\nepoch:1,total_loss:0.35324,accu:0.83865\neval_acc: 0.86980 \nglobal step 330, epoch: 2, batch: 17, loss: 0.27429, accu: 0.89062, speed: 0.23 step/s\nglobal step 360, epoch: 2, batch: 47, loss: 0.11035, accu: 0.96875, speed: 0.46 step/s\nglobal step 390, epoch: 2, batch: 77, loss: 0.07958, accu: 0.98438, speed: 0.46 step/s\nglobal step 420, epoch: 2, batch: 107, loss: 0.22327, accu: 0.93750, speed: 0.46 step/s\nglobal step 450, epoch: 2, batch: 137, loss: 0.35760, accu: 0.85938, speed: 0.46 step/s\nglobal step 480, epoch: 2, batch: 167, loss: 0.23188, accu: 0.93750, speed: 0.46 step/s\nglobal step 510, epoch: 2, batch: 197, loss: 0.12432, accu: 0.96875, speed: 0.46 step/s\nglobal step 540, epoch: 2, batch: 227, loss: 0.15646, accu: 0.92188, speed: 0.46 step/s\nglobal step 570, epoch: 2, batch: 257, loss: 0.14733, accu: 0.95312, speed: 0.46 step/s\nglobal step 600, epoch: 2, batch: 287, loss: 0.13946, accu: 0.96875, speed: 0.46 step/s\nepoch:2,total_loss:0.20175,accu:0.88003\neval_acc: 0.85680 \nglobal step 630, epoch: 3, batch: 4, loss: 0.02362, accu: 1.00000, speed: 0.23 step/s\nglobal step 660, epoch: 3, batch: 34, loss: 0.11228, accu: 0.96875, speed: 0.45 step/s\nglobal step 690, epoch: 3, batch: 64, loss: 0.19188, accu: 0.92188, speed: 0.46 step/s\nglobal step 720, epoch: 3, batch: 94, loss: 0.07411, accu: 0.95312, speed: 0.46 step/s\nglobal step 750, epoch: 3, batch: 124, loss: 0.02254, accu: 1.00000, speed: 0.46 step/s\nglobal step 780, epoch: 3, batch: 154, loss: 0.09569, accu: 0.95312, speed: 0.46 step/s\nglobal step 810, epoch: 3, batch: 184, loss: 0.05387, accu: 0.98438, speed: 0.46 step/s\nglobal step 840, epoch: 3, batch: 214, loss: 0.09059, accu: 0.96875, speed: 0.46 step/s\nglobal step 870, epoch: 3, batch: 244, loss: 0.11632, accu: 0.95312, speed: 0.46 step/s\nglobal step 900, epoch: 3, batch: 274, loss: 0.11151, accu: 0.95312, speed: 0.46 step/s\nglobal step 930, epoch: 3, batch: 304, loss: 0.09381, accu: 0.96875, speed: 0.46 step/s\nepoch:3,total_loss:0.09734,accu:0.90783\neval_acc: 0.87260 \nglobal step 960, epoch: 4, batch: 21, loss: 0.02967, accu: 0.98438, speed: 0.23 step/s\nglobal step 990, epoch: 4, batch: 51, loss: 0.17617, accu: 0.96875, speed: 0.46 step/s\nglobal step 1020, epoch: 4, batch: 81, loss: 0.01692, accu: 1.00000, speed: 0.46 step/s\nglobal step 1050, epoch: 4, batch: 111, loss: 0.02760, accu: 1.00000, speed: 0.46 step/s\nglobal step 1080, epoch: 4, batch: 141, loss: 0.05569, accu: 0.98438, speed: 0.46 step/s\nglobal step 1110, epoch: 4, batch: 171, loss: 0.17499, accu: 0.90625, speed: 0.46 step/s\nglobal step 1140, epoch: 4, batch: 201, loss: 0.04704, accu: 0.98438, speed: 0.46 step/s\nglobal step 1170, epoch: 4, batch: 231, loss: 0.06340, accu: 0.95312, speed: 0.46 step/s\nglobal step 1200, epoch: 4, batch: 261, loss: 0.01554, accu: 0.98438, speed: 0.46 step/s\nglobal step 1230, epoch: 4, batch: 291, loss: 0.03001, accu: 0.98438, speed: 0.46 step/s\nepoch:4,total_loss:0.05630,accu:0.92581\neval_acc: 0.87220 \nglobal step 1260, epoch: 5, batch: 8, loss: 0.02981, accu: 0.98438, speed: 0.23 step/s\nglobal step 1290, epoch: 5, batch: 38, loss: 0.00780, accu: 1.00000, speed: 0.46 step/s\nglobal step 1320, epoch: 5, batch: 68, loss: 0.02140, accu: 0.98438, speed: 0.46 step/s\nglobal step 1350, epoch: 5, batch: 98, loss: 0.14746, accu: 0.96875, speed: 0.46 step/s\nglobal step 1380, epoch: 5, batch: 128, loss: 0.04021, accu: 0.98438, speed: 0.46 step/s\nglobal step 1410, epoch: 5, batch: 158, loss: 0.14581, accu: 0.98438, speed: 0.46 step/s\nglobal step 1440, epoch: 5, batch: 188, loss: 0.03123, accu: 1.00000, speed: 0.46 step/s\nglobal step 1470, epoch: 5, batch: 218, loss: 0.14082, accu: 0.93750, speed: 0.46 step/s\nglobal step 1500, epoch: 5, batch: 248, loss: 0.03275, accu: 1.00000, speed: 0.46 step/s\nglobal step 1530, epoch: 5, batch: 278, loss: 0.00883, accu: 1.00000, speed: 0.46 step/s\nglobal step 1560, epoch: 5, batch: 308, loss: 0.02537, accu: 1.00000, speed: 0.46 step/s\nepoch:5,total_loss:0.03944,accu:0.93806\neval_acc: 0.87440 \n","output_type":"stream"}]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# save_dir = os.path.join(ckpt_dir, \"model_%d\" % 20)\n# if not os.path.exists(save_dir):\n#     os.makedirs(save_dir)\n# torch.save(model.state_dict(),os.path.join(save_dir,'model.pt'))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:47:37.620554Z","iopub.execute_input":"2021-08-03T09:47:37.620969Z","iopub.status.idle":"2021-08-03T09:47:37.625490Z","shell.execute_reply.started":"2021-08-03T09:47:37.620897Z","shell.execute_reply":"2021-08-03T09:47:37.624153Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"model.eval()\ntorch.cuda.empty_cache()\ntest_accuracy = metrics.Accuracy()\ntest_accuracy.to(device)\nfor step,batch in enumerate(test_data_loader):\n    input_ids,token_type_ids,attention_mask,labels = batch\n    input_ids = input_ids.to(device,dtype=torch.long)\n    token_type_ids = token_type_ids.to(device,dtype=torch.long)\n    attention_mask =attention_mask.to(device,dtype=torch.long)\n    labels = labels.to(device,dtype=torch.long)\n    probs = model(input_ids = input_ids, token_type_ids = token_type_ids, attention_mask = attention_mask)\n    batch_acc = test_accuracy(probs, labels)\n    if step != 0 and step % 100 == 0: \n        print(\"eval-> step:%d,batch_acc:%.5f\" % (step,batch_acc))\nprint('!!!!eval-> total_acc:%.5f' % test_accuracy.compute())","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:47:37.627001Z","iopub.execute_input":"2021-08-03T09:47:37.627606Z","iopub.status.idle":"2021-08-03T09:49:19.085667Z","shell.execute_reply.started":"2021-08-03T09:47:37.627567Z","shell.execute_reply":"2021-08-03T09:49:19.084629Z"},"trusted":true},"execution_count":18,"outputs":[{"name":"stdout","text":"eval-> step:100,batch_acc:1.00000\neval-> step:200,batch_acc:0.84375\neval-> step:300,batch_acc:0.84375\neval-> step:400,batch_acc:0.90625\neval-> step:500,batch_acc:0.84375\neval-> step:600,batch_acc:0.87500\neval-> step:700,batch_acc:0.93750\n!!!!eval-> total_acc:0.87148\n","output_type":"stream"}]},{"cell_type":"code","source":"# from transformers import BertTokenizer, BertModel\n# import torch\n\n# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n# model = BertModel.from_pretrained('bert-base-uncased')\n\n# inputs = tokenizer(\"Hello, my dog is cute\", return_tensors=\"pt\")\n# outputs = model(**inputs)\n# last_hidden_states = outputs.pooler_output\n# print(last_hidden_states)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:49:19.087182Z","iopub.execute_input":"2021-08-03T09:49:19.087548Z","iopub.status.idle":"2021-08-03T09:49:19.091802Z","shell.execute_reply.started":"2021-08-03T09:49:19.087509Z","shell.execute_reply":"2021-08-03T09:49:19.090573Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torch.nn as nn\n# import numpy as np\n# from torch.utils.data import Dataset\n# from torch.utils.data import DataLoader\n# class myDataset(Dataset):\n#     def __init__(self,data,label):\n#         super(myDataset).__init__()\n#         self.input_ids = data\n#         self.label = label\n#     def __getitem__(self,idx):\n#         return torch.tensor(self.input_ids[idx], dtype=torch.float),torch.tensor(self.label[idx], dtype=torch.float)\n#     def __len__(self):\n#         return len(self.input_ids)\n# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# model = nn.Linear(1, 1)\n# train_data = np.random.randn(100000).reshape(-1,1)\n# train_label = np.array(train_data * 2 +1)\n# test_data = np.random.randn(100000).reshape(-1,1)\n# test_label = np.array(test_data * 2 +1)\n# train_dataset = myDataset(train_data,train_label)\n# test_dataset = myDataset(test_data,test_label)\n# train_data_loader = DataLoader(\n#                 train_dataset,\n#                 batch_size = 16,\n#                 shuffle=True\n# )\n# test_data_loader = DataLoader(\n#                 test_dataset,\n#                 batch_size = 16,\n#                 shuffle=True\n# )\n# import torch.nn as nn\n# from torch import optim \n# from torch.optim import lr_scheduler\n# criterion = nn.MSELoss()\n# criterion = criterion.to(device)\n# model.to(device)\n# optimizer = optim.Adam(model.parameters(), lr=0.01)\n# scheduler = lr_scheduler.MultiStepLR(optimizer,milestones=[20,80],gamma = 0.9)\n# epoch = 1\n# for i in range(1,epoch+1):\n#     for step,batch in enumerate(train_data_loader):\n#         inputs,label = batch\n#         inputs.to(device)\n#         label.to(device)\n#         model.to(device)\n#         print(inputs[0])\n#         y_hat = model(inputs)\n#         loss = criterion(y_hat,label)\n#         optimizer.zero_grad()\n#         loss.backward()\n#         optimizer.step()\n#         scheduler.step()\n# model.eval()\n# print(model(torch.tensor([[0.1],[1.2]])))","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:49:19.093286Z","iopub.execute_input":"2021-08-03T09:49:19.093652Z","iopub.status.idle":"2021-08-03T09:49:19.103188Z","shell.execute_reply.started":"2021-08-03T09:49:19.093617Z","shell.execute_reply":"2021-08-03T09:49:19.102306Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torchmetrics as metrics\n# class RMSE(metrics.Metric):\n#     def __init__(self):\n#         self.add_state(\"sum_squared_errors\",torch.tensor(0),dist_reduce_fx=\"sum\")\n#         self.add_atate(\"n_observations\",torch.tensor(0),dist_reduce_fx = \"sum\")\n#     def update(self,preds,target):\n#         self.sum_squared_errors += torch.sum((pred - target) ** 2)\n#         self.n_observations += preds.numel()\n#     def compute(self):\n#         return torch.sqrt(self.sum_squared_errors / self.n_observations)","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:49:19.104408Z","iopub.execute_input":"2021-08-03T09:49:19.104839Z","iopub.status.idle":"2021-08-03T09:49:19.115537Z","shell.execute_reply.started":"2021-08-03T09:49:19.104801Z","shell.execute_reply":"2021-08-03T09:49:19.114708Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torchmetrics as metrics\n# train_accuracy = metrics.Accuracy()\n# # train_accuracy.to(device)\n# a = np.random.randn(5,10)\n# b = np.argmax(a,axis = -1)\n# batch_acc = train_accuracy(torch.tensor(a),torch.tensor(b))\n# b -= 1\n# batch_acc = train_accuracy(torch.tensor(a),torch.tensor(b))\n# acc=train_accuracy.compute()\n# print(acc) ","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:49:19.117692Z","iopub.execute_input":"2021-08-03T09:49:19.118965Z","iopub.status.idle":"2021-08-03T09:49:19.128520Z","shell.execute_reply.started":"2021-08-03T09:49:19.118936Z","shell.execute_reply":"2021-08-03T09:49:19.127680Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n","metadata":{"execution":{"iopub.status.busy":"2021-08-03T09:49:19.129800Z","iopub.execute_input":"2021-08-03T09:49:19.130198Z","iopub.status.idle":"2021-08-03T09:49:19.137164Z","shell.execute_reply.started":"2021-08-03T09:49:19.130160Z","shell.execute_reply":"2021-08-03T09:49:19.136452Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}