auto update data sources if they're stale. still needs a refactor

Author: atkinson

docs/notebook.margot.data.ipynb

@@ -55,8 +55,8 @@
     {
      "output_type": "execute_result",
      "data": {
-      "text/plain": "                                SPY                               VTWO  \\\n                          adj_close log_returns realised_vol adj_close   \ndate                                                                     \n2020-06-15 00:00:00+00:00  305.6968    0.009292     0.260969    113.92   \n2020-06-16 00:00:00+00:00  311.5808    0.019065     0.265222    116.76   \n2020-06-17 00:00:00+00:00  310.2865   -0.004163     0.265461    114.60   \n2020-06-18 00:00:00+00:00  310.4060    0.000385     0.264006    114.65   \n2020-06-19 00:00:00+00:00  308.6400   -0.005706     0.263797    113.85   \n\n                                                           margot  \n                          log_returns realised_vol spy_russ_ratio  \ndate                                                               \n2020-06-15 00:00:00+00:00    0.022998     0.433523       2.683434  \n2020-06-16 00:00:00+00:00    0.024624     0.437526       2.668558  \n2020-06-17 00:00:00+00:00   -0.018673     0.442664       2.707561  \n2020-06-18 00:00:00+00:00    0.000436     0.441551       2.707423  \n2020-06-19 00:00:00+00:00   -0.007002     0.441814       2.710935  ",
-      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead tr th {\n        text-align: left;\n    }\n\n    .dataframe thead tr:last-of-type th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr>\n      <th></th>\n      <th colspan=\"3\" halign=\"left\">SPY</th>\n      <th colspan=\"3\" halign=\"left\">VTWO</th>\n      <th>margot</th>\n    </tr>\n    <tr>\n      <th></th>\n      <th>adj_close</th>\n      <th>log_returns</th>\n      <th>realised_vol</th>\n      <th>adj_close</th>\n      <th>log_returns</th>\n      <th>realised_vol</th>\n      <th>spy_russ_ratio</th>\n    </tr>\n    <tr>\n      <th>date</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2020-06-15 00:00:00+00:00</th>\n      <td>305.6968</td>\n      <td>0.009292</td>\n      <td>0.260969</td>\n      <td>113.92</td>\n      <td>0.022998</td>\n      <td>0.433523</td>\n      <td>2.683434</td>\n    </tr>\n    <tr>\n      <th>2020-06-16 00:00:00+00:00</th>\n      <td>311.5808</td>\n      <td>0.019065</td>\n      <td>0.265222</td>\n      <td>116.76</td>\n      <td>0.024624</td>\n      <td>0.437526</td>\n      <td>2.668558</td>\n    </tr>\n    <tr>\n      <th>2020-06-17 00:00:00+00:00</th>\n      <td>310.2865</td>\n      <td>-0.004163</td>\n      <td>0.265461</td>\n      <td>114.60</td>\n      <td>-0.018673</td>\n      <td>0.442664</td>\n      <td>2.707561</td>\n    </tr>\n    <tr>\n      <th>2020-06-18 00:00:00+00:00</th>\n      <td>310.4060</td>\n      <td>0.000385</td>\n      <td>0.264006</td>\n      <td>114.65</td>\n      <td>0.000436</td>\n      <td>0.441551</td>\n      <td>2.707423</td>\n    </tr>\n    <tr>\n      <th>2020-06-19 00:00:00+00:00</th>\n      <td>308.6400</td>\n      <td>-0.005706</td>\n      <td>0.263797</td>\n      <td>113.85</td>\n      <td>-0.007002</td>\n      <td>0.441814</td>\n      <td>2.710935</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
+      "text/plain": "                                SPY                               VTWO  \\\n                          adj_close log_returns realised_vol adj_close   \ndate                                                                     \n2020-06-23 00:00:00+00:00    312.05    0.004593     0.260809  115.2159   \n2020-06-24 00:00:00+00:00    304.09   -0.025840     0.265557  111.4867   \n2020-06-25 00:00:00+00:00    307.35    0.010663     0.259729  113.0700   \n2020-06-26 00:00:00+00:00    300.05   -0.024038     0.269796  110.4600   \n2020-06-29 00:00:00+00:00    304.46    0.014591     0.272220  113.9400   \n\n                                                           margot  \n                          log_returns realised_vol spy_russ_ratio  \ndate                                                               \n2020-06-23 00:00:00+00:00    0.004250     0.429186       2.708394  \n2020-06-24 00:00:00+00:00   -0.032902     0.426147       2.727590  \n2020-06-25 00:00:00+00:00    0.014102     0.411958       2.718228  \n2020-06-26 00:00:00+00:00   -0.023354     0.419855       2.716368  \n2020-06-29 00:00:00+00:00    0.031019     0.425613       2.672108  ",
+      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead tr th {\n        text-align: left;\n    }\n\n    .dataframe thead tr:last-of-type th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr>\n      <th></th>\n      <th colspan=\"3\" halign=\"left\">SPY</th>\n      <th colspan=\"3\" halign=\"left\">VTWO</th>\n      <th>margot</th>\n    </tr>\n    <tr>\n      <th></th>\n      <th>adj_close</th>\n      <th>log_returns</th>\n      <th>realised_vol</th>\n      <th>adj_close</th>\n      <th>log_returns</th>\n      <th>realised_vol</th>\n      <th>spy_russ_ratio</th>\n    </tr>\n    <tr>\n      <th>date</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2020-06-23 00:00:00+00:00</th>\n      <td>312.05</td>\n      <td>0.004593</td>\n      <td>0.260809</td>\n      <td>115.2159</td>\n      <td>0.004250</td>\n      <td>0.429186</td>\n      <td>2.708394</td>\n    </tr>\n    <tr>\n      <th>2020-06-24 00:00:00+00:00</th>\n      <td>304.09</td>\n      <td>-0.025840</td>\n      <td>0.265557</td>\n      <td>111.4867</td>\n      <td>-0.032902</td>\n      <td>0.426147</td>\n      <td>2.727590</td>\n    </tr>\n    <tr>\n      <th>2020-06-25 00:00:00+00:00</th>\n      <td>307.35</td>\n      <td>0.010663</td>\n      <td>0.259729</td>\n      <td>113.0700</td>\n      <td>0.014102</td>\n      <td>0.411958</td>\n      <td>2.718228</td>\n    </tr>\n    <tr>\n      <th>2020-06-26 00:00:00+00:00</th>\n      <td>300.05</td>\n      <td>-0.024038</td>\n      <td>0.269796</td>\n      <td>110.4600</td>\n      <td>-0.023354</td>\n      <td>0.419855</td>\n      <td>2.716368</td>\n    </tr>\n    <tr>\n      <th>2020-06-29 00:00:00+00:00</th>\n      <td>304.46</td>\n      <td>0.014591</td>\n      <td>0.272220</td>\n      <td>113.9400</td>\n      <td>0.031019</td>\n      <td>0.425613</td>\n      <td>2.672108</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
      },
      "metadata": {},
      "execution_count": 4

margot/data/alphavantage.py

@@ -1,31 +1,34 @@
 import os
 import logging
-from datetime import datetime
-import pytz
 
 from alpha_vantage.timeseries import TimeSeries
 
-from margot.data.columns import BaseColumn
+from margot.data.columns import BaseColumn, DailyMixin
 
 logger = logging.getLogger(__name__)
 
 
-class DailyAdjusted(BaseColumn):
-    """A single Symbol time series from AlphaVantage.
+class DailyAdjusted(BaseColumn, DailyMixin):
+    """A daily time series from AlphaVantage.
 
     Example::
 
-        from margot.data.column import alphavantage as av
+        from margot import alphavantage as av
 
-        volume = av.Column(time_series='adjusted_close')
+        volume = av.DailyAdjusted(time_series='adjusted_close')
 
     Args:
-        time_series (str): the name of the time-series that will be returned
+        time_series (str): the name of the time-series that will be returned.
+            Can be one of: 'open', 'high', 'low', 'close', 'adjusted_close',
+            'volume', 'dividend_amount' or 'split_coefficient'.
     """
 
     def clean(self, df):
-        """Clean the df."""
-        # Standardise the column names
+        """
+        Clean the dataframe.
+
+        Alphavantage has odd column names, so we'll fix those.
+        """
         df = df.rename(mapper={
             '1. open': 'open',
             '2. high': 'high',
@@ -38,12 +41,14 @@ def clean(self, df):
         }, axis='columns')
         return super().clean(df)
 
-    def stale(self):
-        now = datetime.now(tz=pytz.UTC)
-        last_complete_trading_day = self.trading_calendar.previous_close()
-
     def fetch(self, symbol: str):
-        """Fetch from remote - this could be the only service specific thing."""
+        """
+        Fetch from remote - this could be the only service specific thing.
+
+        Args:
+            symbol (str): the name of the symbol to fetch
+
+        """
         logger.info('fetching ({}) from alphavantage'.format(symbol))
         ts = TimeSeries(
             key=self.env.get(

margot/data/cboe.py

@@ -2,12 +2,11 @@
 
 import pandas as pd
 
-from margot.data.columns import BaseColumn
-
+from margot.data.columns import BaseColumn, DailyMixin
 logger = logging.getLogger(__name__)
 
 
-class Column(BaseColumn):
+class Column(BaseColumn, DailyMixin):
     """A single OHLC timeiseries from CBOE.
 
     Currently supports the symbols, 'VIX' and 'VIX3M'.

margot/data/columns.py

@@ -1,12 +1,25 @@
 import os
 import logging
+from datetime import datetime
 from pathlib import Path
+
 import pytz
 import pandas as pd
 
 logger = logging.getLogger(__name__)
 
 
+class DailyMixin(object):
+
+    @property
+    def stale(self):
+        """Check if we think there might be new data, and if so - update.
+        """
+        now = pd.Timestamp(datetime.now(tz=pytz.UTC))
+        if self._full_series.index.max().date() < self.trading_calendar.previous_close(now).date():
+            return True
+
+
 class BaseColumn(object):
     """
     BaseColumn is the super class for implementing Columns.
@@ -75,15 +88,15 @@ def clone(self):
     def setup(self, symbol: str, trading_calendar, env: dict):
         """
         Called by Symbol after it is instantiated.
-        
-        Ensures that this column knows the context in which it operates, 
+
+        Ensures that this column knows the context in which it operates,
         including what Symbol we are using and what the trading_calendar is.
         """
         self.symbol = symbol
         self.env = env
         self.trading_calendar = trading_calendar
 
-        # TODO File names should be managed in a central configuration
+        # TODO: File names should be managed in a central configuration
         data_cache = env.get('DATA_CACHE', os.environ.get('DATA_CACHE'))
         Path(data_cache).mkdir(parents=True, exist_ok=True)
 
@@ -107,8 +120,6 @@ def load_or_fetch_series(self, symbol: str):
         In order to return the time-series, first determine if we
         have it and can return it, or if we need to fetch it.
 
-        TODO: Test for up-to-dateness (or maybe that happens in Symbol)?
-
         Args:
             symbol (str): the name of the symbol to fetch.
 
@@ -157,6 +168,9 @@ def series(self):
         """
         if self._full_series is None:
             self._full_series = self.load_or_fetch_series(self.symbol)
+            if self.stale:
+                self.refresh()
+                self.load(self.symbol)
             self._series = self._full_series.copy()
 
         self.INITED = True