adding yaml file to tfrecord encoding

Author: aelbouchti

estimator.py

@@ -9,11 +9,11 @@
 
 import os
 import sys
-from yaml import load, dump
+from yaml import load
 slim = tf.contrib.slim
 
 #Open and read the yaml file:
-stream = open(os.path.join(os.getcwd(), "config_multiclass.yaml"))
+stream = open(os.path.join(os.getcwd(), "yaml","config_multiclass.yaml"))
 data = load(stream)
 
 #=======Dataset Informations=======#
@@ -24,7 +24,8 @@
 gpu_p = data["gpu_p"]
 #Emplacement du checkpoint file
 checkpoint_dir= data["checkpoint_dir"]
-checkpoint_file = os.path.join(checkpoint_dir, "mobilenet_v2_1.4_224.ckpt")
+checkpoint_pattern  = data["checkpoint_pattern"]
+checkpoint_file = os.path.join(checkpoint_dir, checkpoint_pattern)
 ckpt_state = tf.train.get_checkpoint_state(train_dir)
 image_size = data["image_size"]
 #Nombre de classes à prédire

estimator_multiclass.py

@@ -13,18 +13,18 @@
 slim = tf.contrib.slim
 
 #Open and read the yaml file:
-stream = open(os.path.join(os.getcwd(), "config_multilabel.yaml"))
+stream = open(os.path.join(os.getcwd(), "yaml","config_multilabel.yaml"))
 data = load(stream)
-
+stream.close()
 #=======Dataset Informations=======#
 #==================================#
 dataset_dir = data["dataset_dir"]
 train_dir = os.path.join(os.getcwd(), "train")
-summary_dir = os.path.join(train_dir , "summary")
 gpu_p = data["gpu_p"]
 #Emplacement du checkpoint file
 checkpoint_dir= data["checkpoint_dir"]
-checkpoint_file = os.path.join(checkpoint_dir, "mobilenet_v2_1.4_224.ckpt")
+checkpoint_pattern = data["checkpoint_pattern"]
+checkpoint_file = os.path.join(checkpoint_dir, checkpoint_pattern)
 ckpt_state = tf.train.get_checkpoint_state(train_dir)
 image_size = data["image_size"]
 #Nombre de classes à prédire
@@ -55,18 +55,17 @@
 #Create log_dir:
 if not os.path.exists(train_dir):
     os.mkdir(os.path.join(os.getcwd(),train_dir))
-if not os.path.exists(summary_dir):
-    os.mkdir(os.path.join(os.getcwd(),summary_dir))
+
 #===================================================================== Training ===========================================================================#
 #Adding the graph:
 #Set the verbosity to INFO level
 tf.reset_default_graph()
-tf.logging.set_verbosity(tf.logging.INFO)
+tf.logging.set_verbosity(tf.logging.DEBUG)
 
 def input_fn(mode, dataset_dir,file_pattern, file_pattern_for_counting, labels_to_name, batch_size, image_size):
     train_mode = mode==tf.estimator.ModeKeys.TRAIN
     with tf.name_scope("dataset"):
-        dataset = get_dataset_multiclass("eval" if train_mode else "eval",
+        dataset = get_dataset_multiclass("train" if train_mode else "eval",
                                         dataset_dir, file_pattern=file_pattern,
                                         file_pattern_for_counting=file_pattern_for_counting,
                                         labels_to_name=labels_to_name)
@@ -77,7 +76,7 @@ def input_fn(mode, dataset_dir,file_pattern, file_pattern_for_counting, labels_t
 
 def model_fn(features, mode):
     train_mode = mode==tf.estimator.ModeKeys.TRAIN
-    tf.summary.image("images",features['image/encoded'])
+    tf.summary.image("images", features['image/encoded'])
     #Create the model inference
     with slim.arg_scope(mobilenet_v2.training_scope(is_training=train_mode, weight_decay=1e-4, stddev=5e-2, bn_decay=0.99)):
             #TODO: Check mobilenet_v1 module, var "excluding
@@ -98,13 +97,14 @@ def model_fn(features, mode):
     #TODO: Add a func to transform logit tensor to a label-like tensor
     # If value[][class_id]<0.5 then value[][class_id] = 0. else value[][class_id]= 1.
     #It is necessary for a multilabel classification problem
-    
+    logits_sig = tf.nn.sigmoid(logits,name="Sigmoid")
+    logits_sig = tf.to_float(tf.to_int32(logits_sig>=0.5))
     if mode != tf.estimator.ModeKeys.PREDICT:
         metrics = {
-        'Accuracy': tf.metrics.accuracy(features['image/class/id'], logits, name="acc_op"),
-        'Precision': tf.metrics.precision(features['image/class/id'], logits, name="precision_op"),
-        'Recall': tf.metrics.recall(features['image/class/id'], logits, name="recall_op"),
-        'Acc_Class': tf.metrics.mean_per_class_accuracy(features['image/class/id'], logits,len(labels_to_names), name="per_class_acc")
+            'Accuracy': tf.metrics.accuracy(features['image/class/id'], logits_sig, name="acc_op"),
+            'Precision': tf.metrics.precision(features['image/class/id'], logits_sig, name="precision_op"),
+            'Recall': tf.metrics.recall(features['image/class/id'], logits_sig, name="recall_op"),
+            'Acc_Class': tf.metrics.mean_per_class_accuracy(features['image/class/id'], logits_sig, len(labels_to_names), name="per_class_acc")
         }
         for name, value in metrics.items():
             items_list = value[1].get_shape().as_list()

get_tfrec.py

@@ -0,0 +1,49 @@
+import random
+import os
+import tensorflow as tf
+import yaml
+from utils.data_utils import _get_train_valid, _convert_dataset_bis
+
+
+
+#====================================================DEFINE YOUR ARGUMENTS=======================================================================
+stream = open(os.path.join(os.getcwd(), "yaml","config_tfrec.yaml"))
+data = yaml.load(stream)
+print(data)
+dataset_dir = data["dataset_dir"]
+tfrecord_filename = data["tfrecord_filename"]
+validation_size = data["validation_size"]
+num_shards = data["num_shards"]
+class_names_to_ids = data["class_names_to_ids"]
+
+
+def main():
+    #==============================================================CHECKS==========================================================================
+
+    #Check if there is a tfrecord_filename entered
+
+    if not tfrecord_filename:
+        raise ValueError('tfrecord_filename is empty. Please state a tfrecord_filename argument.')
+
+
+
+    #Check if there is a dataset directory entered
+
+    if not dataset_dir:
+        raise ValueError('dataset_dir is empty. Please state a dataset_dir argument.')
+
+    #==============================================================END OF CHECKS===================================================================
+
+    #Get a list of photos filenames like ['123.jpg', '456.jpg'...] and a list of sorted class names from parsing the subdirectories.
+    photos_train, class_train, photos_valid, class_valid = _get_train_valid(dataset_dir)
+    
+    # First, convert the training and validation sets.
+    _convert_dataset_bis('train', photos_train, class_train, class_names_to_ids,
+                     dataset_dir = dataset_dir, tfrecord_filename = tfrecord_filename, batch_size=500, _NUM_SHARDS = num_shards)
+    _convert_dataset_bis('eval', photos_valid, class_valid, class_names_to_ids,
+                     dataset_dir = dataset_dir, tfrecord_filename = tfrecord_filename, batch_size=200, _NUM_SHARDS = num_shards)
+
+    print('\n Finished converting the %s dataset!' % (tfrecord_filename))
+
+if __name__ == "__main__":
+    main()

tfrecord/get_tfrec_csv.py renamed to get_tfrec_csv.py

@@ -2,7 +2,7 @@
 
 import tensorflow as tf
 
-from utils.utils_csv import _dataset_exists, _get_infos, _convert_dataset
+from utils.utils_csv import _dataset_exists, _get_infos, _convert_dataset_multilabel
 
 import pandas as pd
 
@@ -27,22 +27,22 @@
 
 #TODO change this dict into names to ids
 class_names_to_ids = {
-                'No Finding':0, 
-                'Atelectasis':1,
-                'Cardiomegaly':2, 
-                'Effusion':3,
-                'Infiltration':4,
-                'Mass':5,
-                'Nodule':6,
-                'Pneumonia':7,
-                'Pneumothorax':8,
-                'Consolidation':9,
-                'Edema':10,
-                'Emphysema':11,
-                'Fibrosis':12,
-                'Pleural_Thickening':13,
-                'Hernia':14
-                }
+                'No Finding': 0,
+                'Atelectasis' : 1,
+                'Cardiomegaly' : 2, 
+                'Effusion' : 3,
+                'Infiltration' : 4,
+                'Mass' : 5,
+                'Nodule' : 6,
+                'Pneumonia' : 7,
+                'Pneumothorax' : 8,
+                'Consolidation' : 9,
+                'Edema' : 10,
+                'Emphysema' : 11,
+                'Fibrosis' : 12,
+                'Pleural_Thickening' : 13,
+                'Hernia' : 14,
+            }
 
 def main():
     #==============================================================CHECKS==========================================================================
@@ -70,16 +70,17 @@ def main():
     #==============================================================END OF CHECKS===================================================================
     grouped=_get_infos(FLAGS.dataset_dir,"Data_Entry_2017.csv")
     # Divide the training datasets into train and test:(For ChestX like datasets)
+
 
     training_filenames = pd.DataFrame.sample(grouped, frac=(1-FLAGS.validation_size))
-    training_filenames = pd.DataFrame.sample(training_filenames, frac=1,random_state=3)
+    training_filenames = pd.DataFrame.sample(training_filenames, frac=1,random_state=100)
     validation_filenames = grouped.loc[~grouped.index.isin(training_filenames.index), :]
     valid_filenames = pd.DataFrame.sample(validation_filenames, frac=1,random_state=3)
 
     # First, convert the training and validation sets.
-    _convert_dataset('eval', valid_filenames, class_names_to_ids,
+    _convert_dataset_multilabel('eval', valid_filenames, class_names_to_ids,
                      dataset_dir = FLAGS.dataset_dir, tfrecord_filename = FLAGS.tfrecord_filename, _NUM_SHARDS=1)
-    _convert_dataset('train', training_filenames, class_names_to_ids,
+    _convert_dataset_multilabel('train', training_filenames, class_names_to_ids,
                      dataset_dir = FLAGS.dataset_dir, tfrecord_filename = FLAGS.tfrecord_filename, _NUM_SHARDS=FLAGS.num_shards)
 
 

tfrecord/get_tfrec_multiclass.py renamed to get_tfrec_multiclass.py

@@ -26,9 +26,10 @@ def compute_stats_fn(image_data):
                         tf.squeeze(b_mean), tf.squeeze(b_stddev)])
     return result
 
-def computes_stats(sess, images_data, batch_size):
-    images = tf.placeholder(dtype=tf.string, shape=[batch_size])
-    results = tf.map_fn(lambda x: compute_stats_fn(x), images, dtype=tf.float32)
+def computes_stats(sess, images_data):
+    images = tf.placeholder(dtype=tf.string, shape=[None])
+    results = tf.map_fn(lambda x: compute_stats_fn(x), images, dtype=tf.float32,
+                        parallel_iterations=4)
     alpha = sess.run(results, feed_dict={images:images_data})
     GEN_mean, GEN_stddev, R_mean,\
     R_stddev, G_mean, G_stddev, B_mean,\
@@ -164,7 +165,6 @@ class names. Each subdirectory should contain PNG or JPG encoded images.
 
     for root, _ , files in os.walk(dataset_dir):
         path = root.split(os.sep)
-        print(path)
         for file in files:
             photo_filenames.append(os.path.join(root,file))
             class_names.append(path[-1].split("_")[-1])
@@ -246,44 +246,41 @@ def _convert_dataset_bis(split_name, filenames, class_name, class_names_to_ids,
     images_data = []
     class_id_data = []
     assert split_name in ['train', 'eval']
-    max_id = int(math.ceil(len(filenames) / float(batch_size)))
+    lenght = len(filenames)
     output_filename = _get_dataset_filename(
                                 dataset_dir, split_name, tfrecord_filename = tfrecord_filename,stats=False)
     output_filename_stats = _get_dataset_filename(
                                 dataset_dir, split_name, tfrecord_filename = tfrecord_filename,stats=True)
+    tfrecord_stats = tf.python_io.TFRecordWriter(output_filename_stats)
 
     with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer_1:
-        for i in range(len(filenames)):
+        for i in range(lenght):
             # Read the filename:
             image_data = tf.gfile.FastGFile(filenames[i], 'rb').read()
             images_data.append(image_data)
             class_id = class_names_to_ids[class_name[i]]
             class_id_data.append(class_id)
             example_image = image_to_tfexample(image_data, class_id)
             tfrecord_writer_1.write(example_image.SerializeToString())
-    with tf.Graph().as_default():
-        with tf.Session('') as sess:
-            with tf.python_io.TFRecordWriter(output_filename_stats) as tfrecord_writer:
-                for i in range(max_id):
-                    start_ndx = i * batch_size
-                    end_ndx = min((i+1) * batch_size, len(filenames))
-                    try:
+            if (i+1) % batch_size == 0 or i == lenght-1:
+                with tf.Graph().as_default():
+                    with tf.Session('') as sess:
                         gen_mean, gen_stddev, r_mean, r_stddev,\
                         g_mean, g_stddev, b_mean,\
-                        b_stddev = computes_stats(sess, images_data[start_ndx:end_ndx], end_ndx-start_ndx)
+                        b_stddev = computes_stats(sess, images_data)
                         for j in range(len(gen_mean)):
-                            sys.stdout.write('\r>> Converting stats %d/%d shard %d' % (
-                            j+start_ndx, len(filenames), i))
+                            sys.stdout.write('\r>> Converting stats %d/%d' % (
+                            i+1, lenght))
                             sys.stdout.flush()
                             #Py3: use encode("utf-8")
                             example = stats_to_tfexample(gen_mean[j],
                                                         gen_stddev[j], r_mean[j], r_stddev[j],
                                                         g_mean[j], g_stddev[j], b_mean[j],
-                                                        b_stddev[j],class_name[start_ndx+j].encode(),
-                                                        class_id_data[start_ndx+j])
-                            tfrecord_writer.write(example.SerializeToString())
-                    except:
-                        print("batch of image is corrupted")
+                                                        b_stddev[j],class_name[j].encode(),
+                                                        class_id_data[j])
+                            tfrecord_stats.write(example.SerializeToString())
+                images_data = []
+                class_id_data = []   
     sys.stdout.write('\n')
     sys.stdout.flush()
 
@@ -312,7 +309,7 @@ def _convert_dataset_multi(split_name, filenames, class_first_name, class_snd_na
             sys.stdout.flush()
             # Read the filename:
             image_data = tf.gfile.FastGFile(filenames[i], 'rb').read()
-            #TODO/This line is Special to MURA dataset for defining 13 classes.
+            #TODO/The following line is Special to MURA dataset for defining 13 classes.
             class_id = class_names_to_ids[class_snd_name[i]+"_"+class_first_name[i]]
             example_image = image_to_tfexample(image_data, class_id)
             tfrecord_writer.write(example_image.SerializeToString())

rnn/__init__.py

@@ -118,8 +118,10 @@ def process_fn(example):
         return example
     dataset = dataset.map(process_fn)
     if is_training and shuffle:
-        dataset = dataset.shuffle(1000)
-        dataset = dataset.repeat()
-    dataset = dataset.repeat(num_epochs)
+        dataset = dataset.shuffle(2000)
+        dataset = dataset.repeat(-1)
+    else:
+        #Evaluation or test cases:
+        dataset = dataset.repeat(1)
     dataset = dataset.batch(batch_size)
     return dataset