adding changes/code orga

Author: aelbouchti

estimator.py

@@ -2,7 +2,6 @@
 
 
 from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python import debug as tf_debug
 
 import research.slim.nets.nets_factory as nets_factory
 from utils.gen_tfrec import load_batch, get_dataset, load_batch_dense, load_batch_estimator
@@ -14,9 +13,8 @@
 
 #Open and read the yaml file:
 cwd = os.getcwd()
-stream = open(os.path.join(cwd, "yaml","config","config_multiclass.yaml"))
-data = load(stream)
-stream.close()
+with open(os.path.join(cwd, "yaml","config","config.yaml")) as stream:
+    data = load(stream)
 #==================================#
 #=======Dataset Informations=======#
 #==================================#
@@ -64,9 +62,8 @@
 #==================================#
 #=======Network Informations=======#
 #==================================#
-network_file = open(os.path.join(cwd, "yaml", "cnn", model_name+".yaml"))
-network_config = load(network_file)
-network_file.close()
+with open(os.path.join(cwd, "yaml", "cnn", model_name+".yaml")) as network_file:
+    network_config = load(network_file)
 variables_to_exclude = network_config.pop("variables_to_exclude")
 argscope_config = network_config.pop("argscope")
 if "prediction_fn" in network_config.keys():
@@ -173,7 +170,7 @@ def main():
     #Define max steps:
     max_step = num_epochs*num_batches_per_epoch
     #Define configuration non-distributed work:
-    run_config = tf.estimator.RunConfig(model_dir=train_dir, save_checkpoints_steps=num_batches_per_epoch,keep_checkpoint_max=num_epochs)
+    run_config = tf.estimator.RunConfig(save_checkpoints_steps=num_batches_per_epoch,keep_checkpoint_max=num_epochs)
     train_spec = tf.estimator.TrainSpec(input_fn=lambda:input_fn(tf.estimator.ModeKeys.TRAIN,
                                                 dataset_dir, model_name, file_pattern,
                                                 file_pattern_for_counting, names_to_labels,

utils/gen_tfrec.py

@@ -51,7 +51,6 @@ def parse_fn(example):
         }
         parsed_example = tf.parse_single_example(example, feature)
         parsed_example['image/encoded'] = tf.image.decode_image(parsed_example['image/encoded'], channels = 3)
-        parsed_example['image/encoded'] = tf.image.convert_image_dtype(parsed_example['image/encoded'], dtype = tf.float32)
         return parsed_example
     dataset = dataset.map(parse_fn)
     return dataset

utils/images/visu_spark.py

@@ -19,12 +19,17 @@ def load_images(filenames_pattern, train_size=1.):
     train_size : float number representing the train size 
     (use Dataframe.split([train_size, 1 - train_size]))
     """
+    struct_keys = ["origin", "height", "width", "nChannels", "mode", "data"]
+  
     df = spark.read.load(filenames_pattern, format="image")
-    a = df.withColumn("image.data", F.decode(df.image.data,'UTF-8'))\
+    new_cols = [df["image"].getField(alpha) for alpha in struct_keys]
+    new_frame = df.select(*new_cols)
+    a = new_frame.withColumn("image.data", F.decode(new_frame.image.data,'UTF-8'))\
         .drop("image.data")\
-        .withColumnRenamed("image.data", "image.data")
-    a.collect(1)
-    return df
+        .withColumnRenamed("image.data", "data")
+    a.describe().show()
+    a.printSchema()
+    return a
 
 def per_pixel_mean(dataframe):
     """

yaml/config/config.yaml

@@ -1,5 +1,5 @@
 #Dataset directory
-dataset_dir : "D:/MURA-v1.1"
+dataset_dir : "D:/MURA-v1.1/binary"
 #Portion of GPU to attribute for training
 gpu_p : 1.
 #Model name, it serves to target the correct yaml file file in the "cnn" folder: