Add support for binary operators on columns

e.g. foo DIV 1000 - by using “table.foo::DIV::1000”

Author: ndenny

grice/complex_filter.py

@@ -9,7 +9,7 @@
 LIST_FILTERS = ['in', 'not_in', 'bt', 'nbt']
 FILTER_TYPES = ['lt', 'lte', 'eq', 'neq', 'gt', 'gte'] + LIST_FILTERS
 
-ColumnFunction = namedtuple('ColumnFunction', ['table_name', 'column_name', 'func_name'])
+ColumnFunction = namedtuple('ColumnFunction', ['table_name', 'column_name', 'func_name', 'operator_name', 'operator_value'])
 
 
 def _get_column(table_name: str, column_name: str, tables: List[Table]) -> Column:
@@ -33,11 +33,14 @@ def get_column(column_name: str, tables: List[Table]):
     if isinstance(column_name, ColumnFunction):
         func_name = column_name.func_name
         table_name = column_name.table_name
+        operator_name = column_name.operator_name
+        operator_value = column_name.operator_value
         column_name = column_name.column_name
 
     else:
         func_name = None
         table_name = None
+        operator_name = None
 
         try:
             column_name, table_name = column_name.split('.')
@@ -46,8 +49,13 @@ def get_column(column_name: str, tables: List[Table]):
             pass
 
     column = _get_column(table_name, column_name, tables)
+
+    if operator_name:
+        column = column.op(operator_name)(operator_value)
+
     if func_name:
         return getattr(sql_func, func_name)(column)
+
     return column
 
 def parse_filter(filter_string: str):

grice/db_controller.py

@@ -214,11 +214,21 @@ def parse_column_func(column_string):
 
     expected format: column_name
     expected format: function:column_name where function is 'avg' or 'count' etc
+    expected format: function:column_name::operator::value where operator is DIV, + etc
+    expected format: column_name::operator::value
 
     :param sort_string: string
     :return:
     """
     table_name = None
+    clean_vals = [s.strip() for s in column_string.split('::')]
+    if len(clean_vals) == 3:
+        column_string, operator_name, operator_value = clean_vals
+    else:
+        column_string = clean_vals[0]
+        operator_name = None
+        operator_value = None
+
     clean_vals = [s.strip() for s in column_string.split(':')]
     column_name = clean_vals[-1]
     func_name = None
@@ -238,7 +248,7 @@ def parse_column_func(column_string):
         # This means the column name is not in the table_name.column_name format, which is fine.
         pass
 
-    return ColumnFunction(table_name, column_name, func_name)
+    return ColumnFunction(table_name, column_name, func_name, operator_name, operator_value)
 
 def parse_column_funcs(column_list):
     """
@@ -267,7 +277,7 @@ def parse_col_names(column_names):
     :return: column_names: list
     """
     if column_names:
-        clean_cols = (column_name.strip() for column_name in column_names.split(','))
+        clean_cols = (column_name.strip() for column_name in column_names)
         unique_ordered = OrderedDict.fromkeys(clean_cols)
         return list(unique_ordered)
 
@@ -285,7 +295,7 @@ def parse_query_args(query_args):
     sorts = parse_sorts(query_args.getlist('sort'))
     join = parse_join(query_args.get('join'), False) or parse_join(query_args.get('outerjoin'), True)
     column_names = parse_column_funcs(query_args.getlist('columns')) or parse_column_funcs(query_args.get('cols', '').split(','))
-    group_by = parse_col_names(query_args.getlist('group_by', None))
+    group_by = parse_column_funcs(query_args.getlist('group_by', None))
 
     return column_names, page, per_page, filters, sorts, join, group_by
 
@@ -318,7 +328,7 @@ def get_query_args(self):
             sorts = parse_sorts(content.get('sort', []))
             join = parse_join(content.get('join'), False) or parse_join(content.get('outerjoin'), True)
             column_names = parse_column_funcs(content.get('columns', [])) or parse_column_funcs(content.get('cols', '').split(','))
-            group_by = parse_col_names(content.get('group_by', []))
+            group_by = parse_column_funcs(content.get('group_by', []))
             quargs = QueryArguments(column_names, page, per_page, filters, sorts, join, group_by, content.get('_list'))
 
         return quargs

grice/db_service.py

@@ -1,11 +1,13 @@
 import logging
 from collections import namedtuple
+from typing import Union
 import urllib
 
 from sqlalchemy import create_engine, MetaData, Column, Table, select, asc, desc, and_
 from sqlalchemy import engine
 from sqlalchemy.sql import Select
 from sqlalchemy.sql.functions import Function
+from sqlalchemy.sql.expression import BinaryExpression
 from sqlalchemy.engine import reflection
 from grice.complex_filter import ComplexFilter, get_column
 from grice.errors import ConfigurationError, NotFoundError, JoinError
@@ -15,7 +17,7 @@
 DEFAULT_PAGE = 0
 DEFAULT_PER_PAGE = 50
 SORT_DIRECTIONS = ['asc', 'desc']
-SUPPORTED_FUNCS = ['avg', 'count', 'min', 'max', 'sum']
+SUPPORTED_FUNCS = ['avg', 'count', 'min', 'max', 'sum', 'stddev_pop']
 ColumnSort = namedtuple('ColumnSort', ['table_name', 'column_name', 'direction'])
 ColumnPair = namedtuple('ColumnPair', ['from_column', 'to_column'])
 TableJoin = namedtuple('TableJoin', ['table_name', 'column_pairs', 'outer_join'])
@@ -48,15 +50,30 @@ def init_database(db_config):
     return create_engine(eng_url)
 
 
-def function_to_dict(func: Function):
-    data = {
-        'name': str(func),
-        'primary_key': func.primary_key,
-        'table': '<Function {}>'.format(func.name),
-    }
+def computed_column_to_dict(column: Union[Function, BinaryExpression]):
+    """
+    Converts a SqlAlchemy object for a column that contains a computed value to a dict so we can return JSON.
+
+    :param column: a SqlAlchemy Function or a SqlAlchemy BinaryExpression
+    :return: dict
+    """
+    if isinstance(column, Function):
+        data = {
+            'name': str(column),
+            'primary_key': column.primary_key,
+            'table': '<Function {}>'.format(column.name),
+            'type': column.type.__class__.__name__,
+        }
+    elif isinstance(column, BinaryExpression):
+        data = {
+            'name': str(column),
+            'primary_key': column.primary_key,
+            'table': '<BinaryExpression {}>'.format(column),
+            'type': column.type.__class__.__name__,
+        }
     return data
 
-def column_to_dict(column: Column):
+def _column_to_dict(column: Column):
     """
     Converts a SqlAlchemy Column object to a dict so we can return JSON.
 
@@ -80,6 +97,16 @@ def column_to_dict(column: Column):
 
     return data
 
+def column_to_dict(column):
+    """
+    Converts a SqlAlchemy Column, or column-like object to a dict so we can return JSON.
+
+    :param column: a column
+    :return: dict
+    """
+    if isinstance(column, Column):
+        return _column_to_dict(column)
+    return computed_column_to_dict(column)
 
 def table_to_dict(table: Table):
     """
@@ -184,9 +211,7 @@ def apply_group_by(query, table: Table, join_table: Table, group_by: list):
     :return: A SQLAlchemy select object modified to with sorts.
     """
     for group in group_by:
-        column = table.columns.get(group, None)
-        if join_table is not None and not column:
-            column = join_table.columns.get(group, None)
+        column = get_column(group, [table, join_table])
 
         if column is not None:
             query = query.group_by(column)
@@ -305,34 +330,25 @@ def query_table(self, table_name: str, quargs: QueryArguments):  # pylint: disab
             log.debug("Query %s", query)
             result = conn.execute(query)
 
-            for row in result:
-                count_of_map = {}
-                if quargs.format_as_list:
-                    data = []
-                    for column in columns:
-                        if isinstance(column, Function):
-                            counter = count_of_map.get(column.name, 0) + 1
-                            count_of_map[column.name] = counter
-                            column_label = column.name + '_' + str(counter)
-                        else:
-                            column_label = column.table.name + '_' + column.name
-                        data.append(row[column_label])
-                else:
-                    data = {}
-                    for column in columns:
-                        if isinstance(column, Function):
-                            counter = count_of_map.get(column.name, 0) + 1
-                            count_of_map[column.name] = counter
-                            full_column_name = column.name + '_' + str(counter)
-                            column_label = column.name + '_' + str(counter)
-                        else:
-                            full_column_name = column.table.name + '.' + column.name
-                            column_label = column.table.name + '_' + column.name
-                        data[full_column_name] = row[column_label]
-
-                rows.append(data)
-
-        column_data = [column_to_dict(column) if isinstance(column, Column) else function_to_dict(column) for column in columns]
+            if quargs.format_as_list:
+                # SQLalchemy is giving us the data in the correct format
+                rows = result
+            else:
+                column_name_map = {}
+                first_row = True
+                for row in result:
+                    # Make friendlier names if possible
+                    if first_row:
+                        for column, column_label in zip(columns, row.keys()):
+                            if isinstance(column, Column):
+                                full_column_name = column.table.name + '.' + column.name
+                                column_name_map[column_label] = full_column_name
+                        first_row = False
+
+                    data = {column_name_map.get(key, key): val for key, val in row.items()}
+                    rows.append(data)
+
+        column_data = [column_to_dict(column) for column in columns]
 
         return rows, column_data