first commit, WIP

Author: nikolarobottesla

.gitignore

@@ -0,0 +1,31 @@
+#data files
+*.csv
+*.xlsx
+*.xls
+*.json
+
+#log files
+*.log
+tests/*.json
+
+# pycharm project
+.idea/
+# vscode settings
+.vscode/
+# pbs log files
+*.pbs.*
+# local env folder
+env/
+
+# python runtime generated
+__pycache__/
+*.py[cod]
+*$py.class
+
+# files generated by setup.py
+build/
+dist/
+*.egg*/
+
+# file generated by pytest
+*.coverage

cookiecutter_replay.json

@@ -0,0 +1,5 @@
+import os
+
+__version__ = '0.1.0'
+
+os.environ['PLOTME_VERSION'] = __version__

plotme/__init__.py

@@ -0,0 +1,112 @@
+""" Copyright (c) 3M, 2020. All rights reserved.
+helper.py contains data related helper functions including import,
+verification, manipulation, logging, and upload used by roadrunner.py
+"""
+import collections.abc
+import datetime as dt
+import importlib
+import logging
+import os
+from pathlib import Path
+
+
+def start_logging(log_folder='', log_level=logging.INFO, file_name='',
+                  date_time_stamp=''):
+    """
+    Sets up the log file using python's built in logging
+    Parameters
+    ----------
+    log_folder: str
+        relative path to experiment folder
+    log_level: int
+        integer representing logging severity level
+    file_name: str
+        experiment name
+    date_time_stamp: str
+        date + time stamp
+    """
+
+    # in case it is already running shutdown the logging library and reload it
+    # basicConfig only works the 1st time it's called unless you do this
+    logging.shutdown()
+    importlib.reload(logging)
+
+    if len(date_time_stamp) == 0:
+        date_time_stamp = dt.datetime.now().strftime('%Y%m%d_%H.%M.%S')
+    if len(log_folder) > 1:
+        file_name = os.path.join(log_folder, f"{date_time_stamp}_{file_name}.log")
+        Path(file_name).parent.mkdir(parents=True, exist_ok=True)
+    else:
+        file_name = 'log.log'
+    logging_format = '%(asctime)s %(levelname)s %(message)s'
+    logging.basicConfig(filename=file_name, level=log_level, format=logging_format)
+
+    # Get the top-level logger object
+    log = logging.getLogger()
+    # make the log print to the console.
+    console = logging.StreamHandler()
+    log.addHandler(console)
+
+    logging.debug('debug logging active')
+    logging.info('info logging active')
+    logging.warning('warning logging active')
+    logging.error('error logging active')
+
+
+def deep_update(to_update, update):
+    """
+    Update a nested dictionary or similar mapping.
+    Modifies ``source`` in place.
+    Parameters
+    ----------
+    to_update: dict
+        dictionary to update in place
+    update: dict
+        new items for dict
+    Returns
+    -------
+    to_update: dict
+        dictionary with updated values, needed for recursion?
+    """
+    for k, v in update.items():
+        if isinstance(v, collections.abc.Mapping):
+            to_update[k] = deep_update(to_update.get(k, {}), v)
+        else:
+            to_update[k] = v
+    return to_update
+
+
+def try_for(func, args=[], iterations=3):
+    """
+    for loop with try and error handling around any method
+    Parameters
+    ----------
+    func: method
+        function
+    iterations: int
+        number of iterations to run
+    args: tuple
+        arguments
+    Returns
+    -------
+    output: list
+        output objects or variables
+    """
+
+    success = False
+    for x in range(1, iterations + 1):
+        try:
+            output = func(*args)
+            success = True
+            break
+        except Exception as e:
+            logging.error(e, exc_info=True)
+            logging.warning(f'try_for: exception during attempt {x}, trying '
+                            f'{func.__name__}() again')
+
+    if success is False:
+        raise Exception(f"try_for: final attempt to execute {func.__name__}()"
+                        " failed, exiting")
+
+    return output
+

plotme/helper.py

@@ -0,0 +1,150 @@
+import argparse
+import glob
+import json
+import logging
+from pathlib import Path
+import sys
+
+import plotly.graph_objects as go
+import plotly.io as pio
+
+from dirhash import dirhash
+from plotly.subplots import make_subplots
+
+
+import from_xlsx
+import helper
+
+
+def main(kwargs={}):
+    """
+    generate multiple plots, globs through data_root to find plot_info files, checks previous hash against current hash,
+    runs single_plot
+
+    Parameters
+    ----------
+    kwargs: dictionary
+        key word arguments
+
+    """
+
+    plot_info_file = kwargs.get('plot_info', 'plot_info.json')
+
+    data_root = Path(kwargs.get('data_root', Path.home()))
+    # TODO: add setting for depth of glob
+    plot_info_files = data_root.glob(f"**/*{plot_info_file}")
+    for file in plot_info_files:
+        dir_path = file.parent
+
+        # hashing folder recursively
+        current_hash = dirhash(dir_path, "md5", ignore=["previous_hash", "*.html", "*.png"])
+        hash_file_path = Path(dir_path, "previous_hash")
+        if hash_file_path.exists():
+            with open(hash_file_path) as txt_file:
+                previous_hash = txt_file.read()
+        else:
+            previous_hash = ''
+
+        if current_hash == previous_hash:
+            logging.info(f"no changes detected, skipping {dir_path}")
+        else:
+            logging.info(f'loading plot settings from {file}')
+            with open(file) as json_file:
+                plot_info = json.load(json_file)
+            plot_info['plot_dir'] = dir_path
+            single_plot(plot_info)
+            with open(hash_file_path, "w+") as txt_file:
+                txt_file.write(current_hash)
+
+    return 0
+
+
+def single_plot(kwargs={}):
+
+    plot_dir = kwargs.get('plot_dir', Path.home())
+    title = kwargs.get('title', 'plotme plot')
+    x_id = kwargs.get('x_id', 'x_id not set')
+    x_title = kwargs.get('x_title', x_id)  # use x_id if no label is given
+    y_id = kwargs.get('y_id', 'y_id not set')
+    y_title = kwargs.get('y_title', y_id)
+
+    split_on = kwargs.get('split_on', '_')  # used to split x_id out of file name
+    exclude_from_trace_label = kwargs.get('exclude_from_trace_label')  # remove this
+    constant_lines = kwargs.get('constant_lines')
+    constant_lines_x = constant_lines.get('x')  # list
+    constant_lines_y = constant_lines.get('y')  # list
+    add_line = True
+    error_bar_percent_of_y = 5
+    enable_error_bars = False
+
+    folders = glob.glob(f"{plot_dir}/*/")
+    x_dict = {}
+    if add_line:
+        y_dict = {'y=1': [1, 1]}
+    else:
+        y_dict = {}
+    x_max = 0
+    for folder in folders:
+        directory = Path(folder)
+        if directory.name == 'ignore':
+            continue
+        if exclude_from_trace_label not in directory.name:
+            continue
+        else:
+            if exclude_from_trace_label:
+                d_name_part = directory.name.strip(exclude_from_trace_label)
+            else:
+                d_name_part = directory.name
+        x, y = from_xlsx.collect_1_x_per_file(directory, x_id, y_id, split_on)
+        # if not x:
+        #     x, y = collect_from_pkl(directory, x_id, y_id)
+        x_max = max(max(x), x_max)
+        x_dict.update({d_name_part: x})
+        y_dict.update({d_name_part: y})
+
+    if add_line:
+        x_dict['y=1'] = [0, x_max]
+
+    pio.templates.default = "plotly_white"
+    fig = make_subplots(rows=1, cols=1, shared_yaxes=True,
+                        x_title=x_title, y_title=y_title)
+
+    for i, folder in enumerate(x_dict, start=1):
+        if folder == 'y=1':
+            mode = 'lines'
+        else:
+            mode = 'markers'
+        fig.add_trace(go.Scatter(name=folder, mode=mode, x=x_dict[folder], y=y_dict[folder], marker_symbol=i - 1,
+                                 error_y=dict(
+                                     type='percent',  # value of error bar given as percentage of y value
+                                     value=error_bar_percent_of_y,
+                                     visible=enable_error_bars)
+                                 ), row=1, col=1)
+
+    fig.update_layout(height=600, width=1000, title_text=title)
+
+    fig.write_html(str(Path(plot_dir, f"{y_title} vs {x_title}.html")))
+    fig.write_image(str(Path(plot_dir, f"{y_title} vs {x_title}.png")))
+    fig.show()
+
+
+if __name__ == "__main__":
+
+    # parse the arguments
+    parser = argparse.ArgumentParser(description='automates plotting of tabular data')
+
+    parser.add_argument('-s', action="store", default="", type=str,
+                        help="Specify data directory")
+    parser.add_argument('-gt', action="store_true",
+                        help="generate a template")
+
+    args = parser.parse_args()
+
+    helper.start_logging(log_level=logging.INFO)
+    try:
+        main(args)
+        # single_plot()
+    except Exception as e:
+        logging.exception("Fatal error in main")
+        logging.error(e, exc_info=True)
+        sys.exit(1)

plotme/plotme.py

@@ -0,0 +1,23 @@
+[metadata]
+platforms =
+  Linux
+  Windows
+classifiers =
+  Intended Audience :: Developers
+  Operating System :: POSIX :: Linux
+  Operating System :: Microsoft :: Windows
+  Programming Language :: Python
+  Programming Language :: Python :: 3
+  Programming Language :: Python :: 3.7
+  Programming Language :: Python :: 3.8
+  Programming Language :: Python :: 3.9
+  Programming Language :: Python :: 3.10
+  Programming Language :: Python :: Implementation :: CPython
+  Programming Language :: Python :: Implementation :: PyPy
+
+[options.entry_points]
+console_scripts =
+     plotme = plotme.plotme:main
+
+[bdist_wheel]
+universal = 1

setup.cfg

@@ -0,0 +1,4 @@
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path('model/hatchme')))