Merge pull request #9 from ghost/master

OC codegen extensions

Author: bengetch

.travis.yml

@@ -37,7 +37,7 @@ def generate_code(protocol: callable, cfg: CodeGenConfig, mpc_frameworks: list,
 
         # only apply optimizations if required
         if apply_optimizations:
-            dag = comp.rewrite_dag(dag, all_parties=cfg.all_pids, use_leaky_ops=cfg.use_leaky_ops)
+            dag = comp.rewrite_dag(dag, cfg)
 
         # partition into sub-dags that will run in specific frameworks
         mapping = part.heupart(dag, mpc_frameworks, local_frameworks)

code-style.xml

@@ -7,15 +7,6 @@
 from conclave.job import JiffJob
 
 
-'''
-TODO: 
-
-    - append output column row to output csv
-    - agg
-
-'''
-
-
 class JiffCodeGen(CodeGen):
 
     def __init__(self, config, dag: Dag, pid: int,
@@ -120,9 +111,9 @@ def _generate(self, job_name: [str, None], output_directory: str):
             elif isinstance(node, Concat):
                 op_code += self._generate_concat(node)
             elif isinstance(node, Close):
-                op_code += ''
+                op_code += self._generate_close(node)
             elif isinstance(node, Create):
-                op_code += self._generate_create(node)
+                pass
             elif isinstance(node, Join):
                 op_code += self._generate_join(node)
             elif isinstance(node, Open):
@@ -135,13 +126,53 @@ def _generate(self, job_name: [str, None], output_directory: str):
                 op_code += self._generate_divide(node)
             elif isinstance(node, SortBy):
                 op_code += self._generate_sort_by(node)
+            elif isinstance(node, ConcatCols):
+                op_code += self._generate_concat_cols(node)
             elif isinstance(node, Open):
                 op_code += self._generate_open(node)
             else:
                 print("encountered unknown operator type", repr(node))
 
         return self._generate_job(job_name, op_code)
 
+    def _generate_close(self, close_op: Close):
+
+        # node.parent.out_rel.stored_with
+        copied_set = copy.deepcopy(close_op.parent.out_rel.stored_with)
+        data_holder = copied_set.pop()
+
+        template = open(
+            "{0}/create.tmpl".format(self.template_directory), 'r').read()
+
+        data = {
+            "OUTREL": close_op.out_rel.name,
+            "ID": data_holder
+        }
+
+        return pystache.render(template, data)
+
+    def _generate_concat_cols(self, concat_cols_op: ConcatCols):
+
+        if len(concat_cols_op.get_in_rels()) != 2:
+            raise NotImplementedError("Only support concat cols of two relations")
+
+        if concat_cols_op.use_mult:
+
+            template = open(
+                "{0}/matrix_mult.tmpl".format(self.template_directory), 'r').read()
+
+            data = {
+                "LEFT_REL": concat_cols_op.get_in_rels()[0].name,
+                'RIGHT_REL': concat_cols_op.get_in_rels()[1].name,
+                "OUTREL": concat_cols_op.out_rel.name
+            }
+
+            return pystache.render(template, data)
+
+        else:
+            # TODO: implement this
+            return ""
+
     def _generate_create(self, create_op: Create):
 
         # check that the input data belongs to exactly one party
@@ -165,6 +196,23 @@ def _generate_aggregate(self, agg_op: Aggregate):
         if agg_op.aggregator == 'sum':
             template = open(
                 "{}/agg_sum.tmpl".format(self.template_directory), 'r').read()
+        elif agg_op.aggregator == 'mean':
+            template = open(
+                "{}/agg_mean_with_count_col.tmpl".format(self.template_directory), 'r').read()
+
+            data = {
+                "INREL": agg_op.get_in_rel().name,
+                "OUTREL": agg_op.out_rel.name,
+                "KEY_COL": agg_op.group_cols[0].idx,
+                "AGG_COL": agg_op.agg_col.idx,
+                "COUNT_COL": 2
+            }
+
+            return pystache.render(template, data)
+
+        elif agg_op.aggregator == 'std_dev':
+            template = open(
+                "{}/agg_std_dev.tmpl".format(self.template_directory), 'r').read()
         else:
             raise Exception("Unknown aggregator encountered: {}\n".format(agg_op.aggregator))
 

conclave/__init__.py

@@ -70,8 +70,6 @@ def _generate(self, job_name: [str, None], output_directory: [str, None]):
                 op_code += self._generate_divide(node)
             elif isinstance(node, SortBy):
                 op_code += self._generate_sort_by(node)
-            elif isinstance(node, Open):
-                op_code += self._generate_open(node)
             elif isinstance(node, DistinctCount):
                 op_code += self._generate_distinct_count(node)
             elif isinstance(node, Filter):
@@ -393,6 +391,12 @@ def _generate_aggregate(self, agg_op: Aggregate):
         elif agg_op.aggregator == "count":
             template = open(
                 "{}/agg_count.tmpl".format(self.template_directory), 'r').read()
+        elif agg_op.aggregator == 'mean':
+            template = open(
+                "{}/agg_mean_with_count_col.tmpl".format(self.template_directory), 'r').read()
+        elif agg_op.aggregator == "std_dev":
+            template = open(
+                "{}/std_dev.tmpl".format(self.template_directory), 'r').read()
         else:
             raise Exception("Unknown aggregator encountered: {}".format(agg_op.aggregator))
 
@@ -409,7 +413,9 @@ def _generate_aggregate(self, agg_op: Aggregate):
             "OUT_REL": agg_op.out_rel.name,
             "KEY_COL": agg_op.group_cols[0].idx,
             "AGG_COL": agg_op.agg_col.idx,
-            "USE_LEAKY": leaky
+            "USE_LEAKY": leaky,
+            "COUNT_COL": 2,
+            "LEAKY": "Leaky" if leaky else ""
         }
 
         return pystache.render(template, data)

conclave/codegen/jiff.py

@@ -0,0 +1,4 @@
+
+        var {{{OUTREL}}}RESULT = aggregateMean({{{INREL}}}, {{{INREL}}}KeepRows, {{{KEY_COL}}}, {{{AGG_COL}}}, 0);
+        var {{{OUTREL}}} = {{{OUTREL}}}RESULT[0];
+        var {{{OUTREL}}}KeepRows = {{{OUTREL}}}RESULT[1];

conclave/codegen/oblivc.py

@@ -0,0 +1,4 @@
+
+        var {{{OUTREL}}}RESULT = await aggregateMeanWithCountCol({{{INREL}}}, {{{INREL}}}KeepRows, {{{KEY_COL}}}, {{{AGG_COL}}}, {{{COUNT_COL}}}, 0, jiff_instance);
+        var {{{OUTREL}}} = {{{OUTREL}}}RESULT[0];
+        var {{{OUTREL}}}KeepRows = {{{OUTREL}}}RESULT[1];

conclave/codegen/templates/jiff/agg_mean.tmpl

@@ -0,0 +1,4 @@
+
+        var {{{OUTREL}}}RESULT = stdDev({{{INREL}}}, {{{INREL}}}KeepRows, {{{KEY_COL}}}, {{{AGG_COL}}});
+        var {{{OUTREL}}} = {{{OUTREL}}}RESULT[0];
+        var {{{OUTREL}}}KeepRows = {{{OUTREL}}}RESULT[1];

conclave/codegen/templates/jiff/agg_mean_with_count_col.tmpl

@@ -1,4 +1,4 @@
 
-        var {{{OUTREL}}}RESULT = aggregate({{{INREL}}}, {{{INREL}}}KeepRows, {{{KEY_COL}}}, {{{AGG_COL}}});
+        var {{{OUTREL}}}RESULT = await aggregate({{{INREL}}}, {{{INREL}}}KeepRows, {{{KEY_COL}}}, {{{AGG_COL}}}, jiff_instance);
         var {{{OUTREL}}} = {{{OUTREL}}}RESULT[0];
         var {{{OUTREL}}}KeepRows = {{{OUTREL}}}RESULT[1];

conclave/codegen/templates/jiff/agg_std_dev.tmpl

@@ -4,4 +4,4 @@ export NODE_PATH="{{{JIFF_PATH}}}/node_modules"
 
 cd {{{CODE_PATH}}}
 
-node party.js {{{INPUT_PATH}}} {{{PARTY_COUNT}}} default {{{PARTY_ID}}}
+node --max-old-space-size=8192 party.js {{{INPUT_PATH}}} {{{PARTY_COUNT}}} default {{{PARTY_ID}}}

conclave/codegen/templates/jiff/agg_sum.tmpl

@@ -0,0 +1,4 @@
+
+        var {{{OUTREL}}}RESULT = multiplyMatrices({{{LEFT_REL}}}, {{{RIGHT_REL}}}, {{{LEFT_REL}}}KeepRows, {{{RIGHT_REL}}}KeepRows);
+        var {{{OUTREL}}} = {{{OUTREL}}}RESULT[0];
+        var {{{OUTREL}}}KeepRows = {{{OUTREL}}}RESULT[1];