.ignore some file

Author: 刘宇

.gitignore

@@ -138,6 +138,6 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
-.txt
-.json
-.jsonl
+data/
+datasets/
+runs/

项目实战/医疗诊疗对话意图识别挑战赛/BERT-DAC/ERNIE_pretrain/README.md

@@ -0,0 +1 @@
+data form url: https://tianchi.aliyun.com/competition/entrance/532044/information

项目实战/医疗诊疗对话意图识别挑战赛/BERT-DAC/bert_pretrain/README.md

@@ -1,7 +1,6 @@
 import torch
 import torch.nn as nn
-# from pytorch_pretrained_bert import BertModel, BertTokenizer
-from pytorch_pretrained import BertModel, BertTokenizer
+from transformers import BertModel, BertTokenizer
 
 
 class Config(object):

项目实战/医疗诊疗对话意图识别挑战赛/BERT-DAC/data/data.txt

@@ -8,13 +8,12 @@ def load_json(path):
     return data
 
 
-data_dir = '../../../dataset'
-# data_dir = 'dataset'
+data_dir = os.path.join(os.path.dirname(__file__), "data")
 train_set = load_json(os.path.join(data_dir, 'train.json'))
 dev_set = load_json(os.path.join(data_dir, 'dev.json'))
 test_set = load_json(os.path.join(data_dir, 'test.json'))
 
-saved_path = 'THUCNews/data'
+saved_path = os.path.join(os.path.dirname(__file__), "data/process_data")
 os.makedirs(saved_path, exist_ok=True)
 
 tags = [
@@ -33,14 +32,18 @@ def make_tag(path):
             f.write(tag + '\n')
 
 
-def make_data(samples, path):
+def make_data(samples, path, is_train=True):
     out = ''
-    for pid, sample in samples.items():
-        for sent in sample['dialogue']:
+    for pid, sample in samples.items():  # sample is list
+        for sent in sample:
             x = sent['speaker'] + '：' + sent['sentence']
-            assert sent['dialogue_act'] in tag2id
-            y = tag2id.get(sent['dialogue_act'])
-            out += x + '\t' + str(y) + '\n'
+            if is_train:
+                assert sent['dialogue_act'] in tag2id
+                y = tag2id.get(sent['dialogue_act'])
+            else:
+                y = ""
+            out += (x + '\t' + str(y)).strip() + '\n'
+    print(path)
     with open(path, 'w', encoding='utf-8') as f:
         f.write(out)
     return out
@@ -50,4 +53,4 @@ def make_data(samples, path):
 
 make_data(train_set, os.path.join(saved_path, 'train.txt'))
 make_data(dev_set, os.path.join(saved_path, 'dev.txt'))
-make_data(test_set, os.path.join(saved_path, 'test.txt'))
+make_data(test_set, os.path.join(saved_path, 'test.txt'), is_train=False)