Support for different benchmark tooling ( previously only ftsb_redisearch ). (#3)

* [wip] wip on ci integration

* [wip] wip on ci

* [wip] wip on poetry+ci

* [add] adding coverage report

* [add] added badges for CI and Coverage

* [add] added release drafter automation

* [add] enabled different benchmark tools on run. extended testing. kicked off results exporter

Author: filipecosta90

.github/workflows/poetry-pytest.yml

@@ -12,43 +12,43 @@ jobs:
     name: pytest
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-
-    - name: Install Poetry
-      uses: dschep/[email protected]
-
-    - name: Cache Poetry virtualenv
-      uses: actions/cache@v1
-      id: cache
-      with:
-        path: ~/.virtualenvs
-        key: poetry-${{ hashFiles('**/poetry.lock') }}
-        restore-keys: |
-          poetry-${{ hashFiles('**/poetry.lock') }}
-
-    - name: Set Poetry config
-      run: |
-        poetry config virtualenvs.in-project false
-        poetry config virtualenvs.path ~/.virtualenvs
-
-    - name: Install Dependencies
-      run: poetry install
-      if: steps.cache.outputs.cache-hit != 'true'
-
-    - name: Test with pytest
-      env:
-        DJANGO_SETTINGS_MODULE: project.settings
-        SECRETS_FILE: .confidential/ci.json
-      run: poetry run pytest --cov redisbench_admin
-
-    - name: Upload coverage
-      run: poetry run codecov -t ${{ secrets.CODECOV_TOKEN }}
-
-    - name: Publish to PyPI
-      if: github.event_name == 'release'
-      run: |
-        poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }}
+      - uses: actions/checkout@master
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+
+      - name: Install Poetry
+        uses: dschep/[email protected]
+
+      - name: Cache Poetry virtualenv
+        uses: actions/cache@v1
+        id: cache
+        with:
+          path: ~/.virtualenvs
+          key: poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            poetry-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Set Poetry config
+        run: |
+          poetry config virtualenvs.in-project false
+          poetry config virtualenvs.path ~/.virtualenvs
+
+      - name: Install Dependencies
+        run: poetry install
+        if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Test with pytest
+        env:
+          DJANGO_SETTINGS_MODULE: project.settings
+          SECRETS_FILE: .confidential/ci.json
+        run: poetry run pytest --cov redisbench_admin
+
+      - name: Upload coverage
+        run: poetry run codecov -t ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Publish to PyPI
+        if: github.event_name == 'release'
+        run: |
+          poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }}

.github/workflows/release-drafter.yml

@@ -14,6 +14,6 @@ jobs:
       - uses: release-drafter/release-drafter@v5
         with:
           # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml
-           config-name: release-drafter-config.yml
+          config-name: release-drafter-config.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

poetry.lock

@@ -18,6 +18,9 @@ redis = "^3.5.3"
 boto3 = "^1.13.24"
 tqdm = "^4.46.1"
 toml = "^0.10.1"
+seaborn = "^0.10.1"
+matplotlib = "^3.2.1"
+redistimeseries = "^0.8.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^4.6"

pyproject.toml

@@ -1,8 +1,9 @@
 import argparse
 import sys
+
 import toml
-from redisbench_admin import __version__
 
+from redisbench_admin import __version__
 from redisbench_admin.compare.args import create_compare_arguments
 from redisbench_admin.compare.compare import compare_command_logic
 from redisbench_admin.export.args import create_export_arguments

redisbench_admin/cli.py

@@ -1,24 +1,12 @@
-import json
 import os
 import sys
 
 import pandas as pd
 
+from redisbench_admin.utils.results import get_key_results_and_values
 from redisbench_admin.utils.utils import retrieve_local_or_remote_input_json
 
 
-def get_key_results_and_values(baseline_json, step, use_result):
-    selected_run = None
-    metrics = {}
-    if "key-results" in baseline_json and use_result in baseline_json["key-results"]:
-        for name, value in baseline_json["key-results"][step][use_result][0].items():
-            if name == "run-name":
-                selected_run = value
-            else:
-                metrics[name] = value
-    return selected_run, metrics
-
-
 def compare_command_logic(args):
     baseline_file = args.baseline_file
     comparison_file = args.comparison_file
@@ -82,13 +70,17 @@ def compare_command_logic(args):
             if enabled_fail:
                 failing_metrics_serie = df.loc['pct_change'] <= max_negative_pct_change
                 failing_metrics = df.loc['pct_change'][failing_metrics_serie]
-                ammount_of_failing_metrics = len (failing_metrics)
+                ammount_of_failing_metrics = len(failing_metrics)
                 if ammount_of_failing_metrics > 0:
                     df_keys = df.keys()
-                    print( "There was a total of {} metrics that presented a regression above {} %".format(ammount_of_failing_metrics,max_pct_change) )
-                    for pos,failed in enumerate(failing_metrics_serie):
+                    print("There was a total of {} metrics that presented a regression above {} %".format(
+                        ammount_of_failing_metrics, max_pct_change))
+                    for pos, failed in enumerate(failing_metrics_serie):
                         if failed:
-                            print("\tMetric '{}' failed. with an percentage of change of {:.2f} %".format(df_keys[pos],df.loc['pct_change'][pos]))
+                            print("\tMetric '{}' failed. with an percentage of change of {:.2f} %".format(df_keys[pos],
+                                                                                                          df.loc[
+                                                                                                              'pct_change'][
+                                                                                                              pos]))
                     sys.exit(1)
         else:
             print("Skipping step: {} due to command line argument --steps not containing it ({})".format(step, ",".join(
@@ -116,23 +108,3 @@ def generate_comparison_dataframe_configs(benchmark_config, steps):
         step_df_dict[step]["sorting_metric_sorting_direction_map"][metric_name] = False if metric[
                                                                                                "comparison"] == "higher-better" else True
     return step_df_dict
-
-
-def from_resultsDF_to_key_results_dict(resultsDataFrame, step, step_df_dict):
-    key_results_dict = {}
-    key_results_dict["table"] = json.loads(resultsDataFrame.to_json(orient='records'))
-    best_result = resultsDataFrame.head(n=1)
-    worst_result = resultsDataFrame.tail(n=1)
-    first_sorting_col = step_df_dict[step]["sorting_metric_names"][0]
-    first_sorting_median = resultsDataFrame[first_sorting_col].median()
-    result_index = resultsDataFrame[first_sorting_col].sub(first_sorting_median).abs().idxmin()
-    median_result = resultsDataFrame.loc[[result_index]]
-    key_results_dict["best-result"] = json.loads(best_result.to_json(orient='records'))
-    key_results_dict["median-result"] = json.loads(
-        median_result.to_json(orient='records'))
-    key_results_dict["worst-result"] = json.loads(worst_result.to_json(orient='records'))
-    key_results_dict["reliability-analysis"] = {
-        'var': json.loads(resultsDataFrame.var().to_json()),
-        'stddev': json.loads(
-            resultsDataFrame.std().to_json())}
-    return key_results_dict

redisbench_admin/compare/compare.py

@@ -5,4 +5,7 @@ def create_export_arguments(parser):
                         help="comma separated list of steps to be analyzed given the benchmark result files")
     parser.add_argument('--exporter', type=str, default="csv",
                         help="exporter to be used ( either csv or redistimeseries )")
+    parser.add_argument('--use-result', type=str, default="median-result",
+                        help="for each key-metric, use either worst-result, best-result, or median-result")
+    parser.add_argument('--extra-tags', type=str, default="", help='comma separated extra tags in the format of key1=value,key2=value,...')
     return parser