Changing the structure of project.

Author: ldulcic

root/constants.py renamed to constants.py

@@ -2,16 +2,16 @@
 from collections import namedtuple
 from keras.utils import to_categorical
 from keras_preprocessing.sequence import pad_sequences
-from root.constants import NO_ENTITY_TOKEN, MAX_LEN, PAD
+from constants import NO_ENTITY_TOKEN, MAX_LEN, PAD
 from .data_processor import numericalize
 from .vocab import TextVocab, LabelVocab, PosVocab, CharacterVocab
 
 
 def load_dataset():
     # load examples
-    train_examples = load_examples('../dataset/raw/train.txt')
-    val_examples = load_examples('../dataset/raw/valid.txt')
-    test_examples = load_examples('../dataset/raw/test.txt')
+    train_examples = load_examples('data/raw/train.txt')
+    val_examples = load_examples('data/raw/valid.txt')
+    test_examples = load_examples('data/raw/test.txt')
 
     # build vocabularies
     text_vocab = TextVocab.build(list(map(lambda e: e.sentence, train_examples)))
@@ -30,7 +30,7 @@ def load_examples(file_path):
     """
     Loads sentences from file in CoNLL 2003 format.
 
-    :param file_path: Path to file with CoNLL data.
+    :param file_path: Path to file with CoNLL dataset.
     :return: list(Example)
     """
     examples = []

dataset/csv/test.csv renamed to data/csv/test.csv

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from collections import defaultdict, Counter
-from root.constants import PAD, UNK
+from constants import PAD, UNK
 
 
 class Vocab(ABC):

dataset/csv/train.csv renamed to data/csv/train.csv

@@ -1,7 +1,7 @@
 import sys
 import numpy as np
 
-GLOVE_DIR = '../embedding/glove.6B.100d.txt'
+GLOVE_DIR = 'embedding/glove.6B.100d.txt'
 
 
 def get_pretrained_glove(num_words, text_vocab):

dataset/csv/valid.csv renamed to data/csv/valid.csv

@@ -7,7 +7,7 @@
 from keras.utils.vis_utils import plot_model
 from keras.callbacks import TensorBoard
 
-from root.constants import MAX_LEN
+from constants import MAX_LEN
 
 
 class NeuralNetwork(object):
@@ -64,7 +64,7 @@ def train(self, epochs, embedding=None):
 
         model.compile(optimizer="rmsprop", loss='categorical_crossentropy', metrics=['accuracy'])
 
-        plot_model(model, to_file='../models/ner_model_image.png')
+        plot_model(model, to_file='models/ner_model_image.png')
         print(model.summary())
 
         model.compile(optimizer="rmsprop", metrics=['accuracy'], loss='categorical_crossentropy')
@@ -93,16 +93,16 @@ def train(self, epochs, embedding=None):
 
 
 def create_dir():
-    runs = ([x[0] for x in os.walk("../results/logs")])
+    runs = ([x[0] for x in os.walk("results/logs")])
     runs = [x for x in runs if "run" in x]
     runs = list(map(int, re.findall(r'\d+', "".join(runs))))
     runs.sort()
     if len(runs) == 0:
-        return "../results/logs/run1"
+        return "results/logs/run1"
 
     dir_idx = runs[-1] + 1
 
-    dir = "../results/logs/run" + str(dir_idx)
+    dir = "results/logs/run" + str(dir_idx)
 
     if not os.path.exists(dir):
         os.makedirs(dir)

dataset/dataset.py renamed to data/dataset.py

@@ -1,16 +1,22 @@
+import os
 import argparse
 import spacy
 import numpy as np
 from keras.models import load_model
-from root.dataset.data_processor import numericalize
+from dataset.data_processor import numericalize
 from utils.serialization import load_object
-from root.constants import NO_ENTITY_TOKEN
+from constants import NO_ENTITY_TOKEN
 
 
 def parse_args():
     parser = argparse.ArgumentParser(description='Script for using NER model')
     parser.add_argument('-p', '--path', help='Path to model and vocabulary directory.')
-    return parser.parse_args()
+
+    args = parser.parse_args()
+    # add path separator (/) at the end if needed
+    args.path = args.path if args.path[-1] == os.path.sep else args.path + os.path.sep
+
+    return args
 
 
 def main():

dataset/raw/test.txt renamed to data/raw/test.txt

@@ -5,7 +5,7 @@
 from keras.models import load_model
 from sklearn.metrics import confusion_matrix, precision_recall_fscore_support
 
-from root.constants import MAX_LEN
+from constants import MAX_LEN
 from utils.classification_report import classification_report
 from utils.plot_confusion_matrix_util import plot_confusion_matrix
 
@@ -41,13 +41,13 @@ def test_model(model_path, test, text_vocab, labels_vocab):
     print(report)
 
     # plot_classification_report(report)
-    # plt.savefig('../results/classification_report.png', dpi=200, format='png', bbox_inches='tight')
+    # plt.savefig('results/classification_report.png', dpi=200, format='png', bbox_inches='tight')
     # plt.close()
 
     # Confusion Matrix
     cnf_matrix = confusion_matrix(true_values, predicted_values)
     np.set_printoptions(precision=2)
     # TODO fix classes
     plot_confusion_matrix(cnf_matrix, classes=list(labels_vocab.stoi.keys()), normalize=True, title='Normalized confusion matrix')
-    plt.savefig('../results/confusion_matrix.png', dpi=200, format='png', bbox_inches='tight')
+    plt.savefig('results/confusion_matrix.png', dpi=200, format='png', bbox_inches='tight')
     plt.close()

dataset/raw/train.txt renamed to data/raw/train.txt

@@ -1,9 +1,9 @@
 import matplotlib.pyplot as plt
 
 from embedding.glove import get_pretrained_glove
-from root.dataset.api import load_dataset
-from root.model import NeuralNetwork
-from root.test_model import test_model
+from dataset.api import load_dataset
+from model import NeuralNetwork
+from test_model import test_model
 from datetime import datetime
 from utils.serialization import save_object
 
@@ -15,7 +15,7 @@
 num_chars = len(character_vocab.itos)
 
 # save vocabulary
-save_path = '../models/' + datetime.now().strftime("%Y-%m-%d-%H:%M") + '/'
+save_path = 'models/' + datetime.now().strftime("%Y-%m-%d-%H:%M") + '/'
 save_object(text_vocab, save_path + 'text_vocab')
 save_object(labels_vocab, save_path + 'labels_vocab')
 
@@ -34,7 +34,7 @@
 plt.ylabel('Accuracy')
 plt.xlabel('Epoch')
 plt.legend(['train', 'validation'], loc='lower right')
-plt.savefig('../results/model_accuracy.png', dpi=200, format='png', bbox_inches='tight')
+plt.savefig('results/model_accuracy.png', dpi=200, format='png', bbox_inches='tight')
 plt.close()
 
 # Plot loss
@@ -44,5 +44,5 @@
 plt.ylabel('Loss')
 plt.xlabel('Epoch')
 plt.legend(['train', 'validation'], loc='upper right')
-plt.savefig('../results/model_loss.png', dpi=200, format='png', bbox_inches='tight')
+plt.savefig('results/model_loss.png', dpi=200, format='png', bbox_inches='tight')
 plt.close()

dataset/raw/valid.txt renamed to data/raw/valid.txt

@@ -24,7 +24,7 @@ def classification_report(y_true, y_pred, labels=None, target_names=None,
     digits : int
         Number of digits for formatting output floating point values
     average : string, ['weighted' (default), 'binary', 'micro', 'macro']
-        Determines the type of averaging performed on the data, after reporting the individual results per class:
+        Determines the type of averaging performed on the dataset, after reporting the individual results per class:
         ``'binary'``:
             Only report results for the class specified by ``pos_label``.
             This is applicable only if targets (``y_{true,pred}``) are binary.