Fix optimizer for Reader training (#307)

* fix optimizer for Reader training

* bump version

* tmp fix for lr schedule

* fixed train and updated tuto

Author: andrelmfarias

cdqa/reader/bertqa_sklearn.py

@@ -31,14 +31,15 @@
 import torch
 from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
 from torch.utils.data.distributed import DistributedSampler
+from torch.optim.lr_scheduler import LambdaLR
 from tqdm.autonotebook import tqdm, trange
 
 from transformers import PYTORCH_PRETRAINED_BERT_CACHE, WEIGHTS_NAME, CONFIG_NAME
 
 from transformers import BertForQuestionAnswering, DistilBertForQuestionAnswering
 from transformers import BertConfig, DistilBertConfig
 from transformers import BertTokenizer, DistilBertTokenizer
-from transformers import AdamW, WarmupLinearSchedule
+from transformers import AdamW
 from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
 
 from sklearn.base import BaseEstimator, TransformerMixin
@@ -934,6 +935,16 @@ def _n_best_predictions(final_predictions_sorted, n):
         final_prediction_list.append(curr_pred)
     return final_prediction_list
 
+def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1):
+    """ Create a schedule with a learning rate that decreases linearly after
+    linearly increasing during a warmup period.
+    """
+    def lr_lambda(current_step):
+        if current_step < num_warmup_steps:
+            return float(current_step) / float(max(1, num_warmup_steps))
+        return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps)))
+
+    return LambdaLR(optimizer, lr_lambda, last_epoch)
 
 class BertProcessor(BaseEstimator, TransformerMixin):
     """
@@ -1052,6 +1063,10 @@ class BertQA(BaseEstimator):
     warmup_proportion : float, optional
         Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10%%
         of training. (the default is 0.1)
+    warmup_steps : int, optional
+        Linear warmup over warmup_steps.
+    adam_epsilon : float
+        Epsilon for Adam optimizer. (default: 1e-8)
     n_best_size : int, optional
         The total number of n-best predictions to generate in the nbest_predictions.json
         output file. (the default is 20)
@@ -1122,6 +1137,8 @@ def __init__(
         learning_rate=5e-5,
         num_train_epochs=3.0,
         warmup_proportion=0.1,
+        warmup_steps=0,
+        adam_epsilon=1e-8,
         n_best_size=20,
         max_answer_length=30,
         verbose_logging=False,
@@ -1145,6 +1162,8 @@ def __init__(
         self.learning_rate = learning_rate
         self.num_train_epochs = num_train_epochs
         self.warmup_proportion = warmup_proportion
+        self.warmup_steps = warmup_steps
+        self.adam_epsilon = adam_epsilon
         self.n_best_size = n_best_size
         self.max_answer_length = max_answer_length
         self.verbose_logging = verbose_logging
@@ -1344,12 +1363,8 @@ def fit(self, X, y=None):
                 warmup=self.warmup_proportion, t_total=num_train_optimization_steps
             )
         else:
-            optimizer = BertAdam(
-                optimizer_grouped_parameters,
-                lr=self.learning_rate,
-                warmup=self.warmup_proportion,
-                t_total=num_train_optimization_steps,
-            )
+            optimizer = AdamW(optimizer_grouped_parameters, lr=self.learning_rate, eps=self.adam_epsilon)
+            scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=self.warmup_steps, num_training_steps=num_train_optimization_steps)
 
         self.model.train()
         for _ in trange(int(self.num_train_epochs), desc="Epoch"):
@@ -1364,12 +1379,14 @@ def fit(self, X, y=None):
                     batch = tuple(
                         t.to(self.device) for t in batch
                     )  # multi-gpu does scattering it-self
-                input_ids, input_mask, segment_ids, start_positions, end_positions = (
-                    batch
-                )
-                loss = self.model(
-                    input_ids, segment_ids, input_mask, start_positions, end_positions
-                )
+                inputs = {'input_ids':       batch[0],
+                          'attention_mask':  batch[1],
+                          'start_positions': batch[3],
+                          'end_positions':   batch[4]}
+                if 'distilbert' not in self.bert_model:
+                    inputs['token_type_ids'] = batch[2]
+                outputs = self.model(**inputs)
+                loss = outputs[0]
                 if self.n_gpu > 1:
                     loss = loss.mean()  # mean() to average on multi-gpu.
                 if self.gradient_accumulation_steps > 1:
@@ -1389,6 +1406,7 @@ def fit(self, X, y=None):
                         for param_group in optimizer.param_groups:
                             param_group["lr"] = lr_this_step
                     optimizer.step()
+                    scheduler.step()  # Update learning rate schedule
                     optimizer.zero_grad()
                     global_step += 1
 

examples/tutorial-train-reader-squad.ipynb

@@ -35,10 +35,9 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/home/supercalculateur/source/andre/cdqa-dev/env-cdqa/lib/python3.6/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
-      "  warnings.warn(msg, category=DeprecationWarning)\n",
       "/home/supercalculateur/source/andre/cdqa-dev/env-cdqa/lib/python3.6/site-packages/tqdm/autonotebook/__init__.py:18: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
-      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n"
+      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n",
+      "I1120 11:43:47.615704 140657575868224 file_utils.py:39] PyTorch version 1.2.0 available.\n"
      ]
     }
    ],
@@ -99,11 +98,20 @@
     "ExecuteTime": {
      "end_time": "2019-07-20T13:58:36.512980Z",
      "start_time": "2019-07-20T13:46:44.792080Z"
-    }
+    },
+    "collapsed": true
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "I1120 11:43:48.194295 140657575868224 tokenization_utils.py:375] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/supercalculateur/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
+     ]
+    }
+   ],
    "source": [
-    "train_processor = BertProcessor(do_lower_case=True, is_training=True, n_jobs=-1)\n",
+    "train_processor = BertProcessor(do_lower_case=True, is_training=True)\n",
     "train_examples, train_features = train_processor.fit_transform(X='./data/SQuAD_1.1/train-v1.1.json')"
    ]
   },
@@ -116,9 +124,109 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "I1120 11:43:53.164162 140657575868224 configuration_utils.py:152] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json from cache at /home/supercalculateur/.cache/torch/transformers/distributed_-1/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.bf3b9ea126d8c0001ee8a1e8b92229871d06d36d8808208cc2449280da87785c\n",
+      "I1120 11:43:53.165523 140657575868224 configuration_utils.py:169] Model config {\n",
+      "  \"attention_probs_dropout_prob\": 0.1,\n",
+      "  \"finetuning_task\": null,\n",
+      "  \"hidden_act\": \"gelu\",\n",
+      "  \"hidden_dropout_prob\": 0.1,\n",
+      "  \"hidden_size\": 768,\n",
+      "  \"initializer_range\": 0.02,\n",
+      "  \"intermediate_size\": 3072,\n",
+      "  \"is_decoder\": false,\n",
+      "  \"layer_norm_eps\": 1e-12,\n",
+      "  \"max_position_embeddings\": 512,\n",
+      "  \"num_attention_heads\": 12,\n",
+      "  \"num_hidden_layers\": 12,\n",
+      "  \"num_labels\": 2,\n",
+      "  \"output_attentions\": false,\n",
+      "  \"output_hidden_states\": false,\n",
+      "  \"output_past\": true,\n",
+      "  \"pruned_heads\": {},\n",
+      "  \"torchscript\": false,\n",
+      "  \"type_vocab_size\": 2,\n",
+      "  \"use_bfloat16\": false,\n",
+      "  \"vocab_size\": 30522\n",
+      "}\n",
+      "\n",
+      "I1120 11:43:53.591548 140657575868224 modeling_utils.py:383] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin from cache at /home/supercalculateur/.cache/torch/transformers/distributed_-1/aa1ef1aede4482d0dbcd4d52baad8ae300e60902e88fcb0bebdec09afd232066.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157\n",
+      "I1120 11:43:55.430284 140657575868224 modeling_utils.py:453] Weights of BertForQuestionAnswering not initialized from pretrained model: ['qa_outputs.weight', 'qa_outputs.bias']\n",
+      "I1120 11:43:55.431005 140657575868224 modeling_utils.py:456] Weights from pretrained model not used in BertForQuestionAnswering: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9fb44d7dc6854474a6dc36ea50168573",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, description='Epoch', max=2, style=ProgressStyle(description_width='initial…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f50802fcf50043f1bfa008c9b911d3df",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, description='Iteration', max=4, style=ProgressStyle(description_width='ini…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c8eaa69941804829bc7c2c984487f7d2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, description='Iteration', max=4, style=ProgressStyle(description_width='ini…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "BertQA(adam_epsilon=1e-08, bert_model='bert-base-uncased', do_lower_case=True,\n",
+       "       fp16=False, gradient_accumulation_steps=1, learning_rate=3e-05,\n",
+       "       local_rank=-1, loss_scale=0, max_answer_length=30, n_best_size=20,\n",
+       "       no_cuda=False, null_score_diff_threshold=0.0, num_train_epochs=2,\n",
+       "       output_dir='models', predict_batch_size=8, seed=42, server_ip='',\n",
+       "       server_port='', train_batch_size=12, verbose_logging=False,\n",
+       "       version_2_with_negative=False, warmup_proportion=0.1, warmup_steps=0)"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "reader = BertQA(train_batch_size=12,\n",
     "                learning_rate=3e-5,\n",

requirements.txt

@@ -3,7 +3,7 @@ flask_cors==3.0.8
 joblib==0.13.2
 pandas==0.25.0
 prettytable==0.7.2
-transformers==2.1.1
+transformers>=2.1.1
 scikit_learn==0.21.2
 tika==1.19
 torch>=1.2.0

setup.py

@@ -8,7 +8,7 @@ def read(file):
 
 setup(
     name="cdqa",
-    version="1.3.6",
+    version="1.3.7",
     author="Félix MIKAELIAN, André FARIAS, Matyas AMROUCHE, Olivier SANS, Théo NAZON",
     description="An End-To-End Closed Domain Question Answering System",
     long_description=read("README.md"),