minor tweks

Author: toddbirchard

biquery_sql_etl/__init__.py

@@ -2,26 +2,19 @@
 from biquery_sql_etl.engines import bigquery_engine, rdbms_engine
 from biquery_sql_etl.queries import sql_queries
 from biquery_sql_etl.client import DataClient
-from config import bigquery_table
 
 
 logger.add('logs/queries.log', format="{time} {message}", level="INFO")
 
 
 def init_pipeline():
     """Move data between Bigquery and MySQL."""
-    total_rows_affected = 0
-    bqc, dbc = data_sources()
-    for table_name, query in sql_queries.items():
-        fetched_rows = bqc.fetch_rows(query, table=bigquery_table)
-        inserted_rows = dbc.insert_rows(fetched_rows, table_name, replace=True)
-        logger.info(inserted_rows)
-        total_rows_affected += len(fetched_rows)
-    logger.info(f"Completed migration of {total_rows_affected} rows from BigQuery to MySQL.")
-
-
-def data_sources():
-    """Construct datasources."""
+    num_rows = 0
     bqc = DataClient(bigquery_engine)
     dbc = DataClient(rdbms_engine)
-    return bqc, dbc
+    for table_name, query in sql_queries.items():
+        rows = bqc.fetch_rows(query)
+        insert = dbc.insert_rows(rows, table_name, replace=True)
+        logger.info(insert)
+        num_rows += len(rows)
+    logger.info(f"Completed migration of {num_rows} rows from BigQuery to MySQL.")

biquery_sql_etl/client.py

@@ -23,7 +23,7 @@ def insert_rows(self, rows, table=None, replace=None):
         self.engine.execute(self.table.insert(), rows)
         return self.construct_response(rows, table)
 
-    def fetch_rows(self, query, table=None):
+    def fetch_rows(self, query):
         """Fetch all rows via query."""
         rows = self.engine.execute(query).fetchall()
         return rows

logs/queries.log

@@ -1 +1,4 @@
 # QUERY HISTORY
+2020-02-11T07:29:26.499248-0500 Inserted 100 rows into `weekly_stats` with 4 columns: title, url, slug, views
+2020-02-11T07:29:28.552129-0500 Inserted 500 rows into `monthly_stats` with 4 columns: title, url, slug, views
+2020-02-11T07:29:28.552409-0500 Completed migration of 600 rows from BigQuery to MySQL.