add spark changes

Author: aelbouchti

estimator.py

@@ -68,9 +68,7 @@
 network_config = load(network_file)
 network_file.close()
 variables_to_exclude = network_config.pop("variables_to_exclude")
-print(variables_to_exclude)
 argscope_config = network_config.pop("argscope")
-print(argscope_config)
 if "prediction_fn" in network_config.keys():
     network_config["prediction_fn"] = getattr(tf.contrib.layers, network_config["prediction_fn"])
 if "activation_fn" in network_config.keys():
@@ -103,7 +101,6 @@ def input_fn(mode, dataset_dir, model_name, file_pattern, file_pattern_for_count
 
 def model_fn(features, mode):
     train_mode = mode==tf.estimator.ModeKeys.TRAIN
-    tf.summary.image("images",features['image/encoded'])
     tf.summary.histogram("final_image_hist", features['image/encoded'])
     #Create the model structure using network_fn :
     network = nets_factory.networks_map[model_name]

estimator_multilabel.py

@@ -2,7 +2,7 @@
 
 from tensorflow.python.platform import tf_logging as logging
 import research.slim.nets.nets_factory as nets_factory
-from utils.gen_tfrec import get_dataset_multiclass, load_batch_estimator
+from utils.gen_tfrec import get_dataset_multilabel, load_batch_estimator
 
 import os
 import sys
@@ -65,7 +65,7 @@
 def input_fn(mode, dataset_dir,file_pattern, file_pattern_for_counting, labels_to_name, batch_size, image_size):
     train_mode = mode==tf.estimator.ModeKeys.TRAIN
     with tf.name_scope("dataset"):
-        dataset = get_dataset_multiclass("train" if train_mode else "eval",
+        dataset = get_dataset_multilabel("train" if train_mode else "eval",
                                         dataset_dir, file_pattern=file_pattern,
                                         file_pattern_for_counting=file_pattern_for_counting,
                                         labels_to_name=labels_to_name)

test.py

@@ -0,0 +1,11 @@
+import tensorflow as tf
+
+x = tf.placeholder(tf.float32, name="input")
+# y =a*x + b 
+a = tf.constant(2., dtype=tf.float32)
+b = tf.constant(0.5,dtype=tf.float32 )
+y = a*x+b
+
+with tf.Session() as sess:
+    c = sess.run([y], feed_dict={'input:0' : 0.2})
+    print(c)

utils/gen_tfrec.py

@@ -30,7 +30,7 @@ def parse_fn(example):
     dataset = dataset.map(parse_fn, num_parallel_calls=8)
     return dataset
 
-def get_dataset_multiclass(phase_name, dataset_dir, file_pattern, file_pattern_for_counting, labels_to_name):
+def get_dataset_multilabel(phase_name, dataset_dir, file_pattern, file_pattern_for_counting, labels_to_name):
     """Creates dataset based on phased_name(train or evaluation), datatset_dir. """
 
     #On vérifie si phase_name est 'train' ou 'validation'

utils/images/visu_spark.py

@@ -21,8 +21,9 @@ def load_images(filenames_pattern, train_size=1.):
     """
     df = spark.read.load(filenames_pattern, format="image")
     a = df.withColumn("image.data", F.decode(df.image.data,'UTF-8'))\
-        .drop("image.data").withColumnRenamed("image.data", "image.data")
-    a["image.data"].show(1)     
+        .drop("image.data")\
+        .withColumnRenamed("image.data", "image.data")
+    a.collect(1)
     return df
 
 def per_pixel_mean(dataframe):

yaml/config/config_multiclass.yaml

@@ -3,7 +3,7 @@ dataset_dir : "D:/MURA-v1.1/multiclass"
 #Portion of GPU to attribute for training
 gpu_p : 1.
 
-#Model name to call automatically:
+#Model name to call automatically:******
 model_name : "mobilenet_v2"
 variables_to_exclude : []
 #Checkpoint directory (For transfer learning)