Improved run-local and run-remote logic. Checking for errors in intermediate steps (#45)

* [add] Improved run-local and run-remote logic. Checking for errors in intermediate steps

* [add] Changed Readme to point towards redismodules dir

Author: filipecosta90

README.md

@@ -1,8 +1,8 @@
-[![codecov](https://codecov.io/gh/filipecosta90/redisbench-admin/branch/master/graph/badge.svg)](https://codecov.io/gh/filipecosta90/redisbench-admin)
-![Actions](https://github.com/filipecosta90/redisbench-admin/workflows/Run%20Tests/badge.svg?branch=master)
+[![codecov](https://codecov.io/gh/RedisLabsModules/redisbench-admin/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisLabsModules/redisbench-admin)
+![Actions](https://github.com/RedisLabsModules/redisbench-admin/workflows/Run%20Tests/badge.svg?branch=master)
 ![Actions](https://badge.fury.io/py/redisbench-admin.svg)
 
-# redisbench-admin
+# [redisbench-admin](https://github.com/RedisLabsModules/redisbench-admin) 
 
 Redis benchmark run helper can help you with the following tasks:
 - Setup abd teardown of benchmarking infrastructure specified on [RedisLabsModules/testing-infrastructure](https://github.com/RedisLabsModules/testing-infrastructure)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.1.44"
+version = "0.1.45"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>"]
 readme = "README.md"

redisbench_admin/cli.py

@@ -12,8 +12,6 @@
 from redisbench_admin.export.export import export_command_logic
 from redisbench_admin.extract.args import create_extract_arguments
 from redisbench_admin.extract.extract import extract_command_logic
-from redisbench_admin.run.args import create_run_arguments
-from redisbench_admin.run.run import run_command_logic
 from redisbench_admin.run_local.args import create_run_local_arguments
 from redisbench_admin.run_local.run_local import run_local_command_logic
 from redisbench_admin.run_remote.args import create_run_remote_arguments
@@ -34,13 +32,15 @@ def populate_with_poetry_data():
 
     return project_name, project_description, project_version
 
+
 # logging settings
 logging.basicConfig(
     format="%(asctime)s %(levelname)-4s %(message)s",
     level=logging.INFO,
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
+
 def main():
     tool = None
     if len(sys.argv) < 2:

redisbench_admin/export/common/common.py

@@ -22,7 +22,7 @@ def prepare_tags(k):
 
 
 def add_datapoint(
-    time_series_dict, broader_ts_name, start_time_ms, metric_value, tags_array
+        time_series_dict, broader_ts_name, start_time_ms, metric_value, tags_array
 ):
     if broader_ts_name not in time_series_dict:
         tags_dict = {}
@@ -74,14 +74,14 @@ def get_or_None(dict, property):
 
 
 def get_kv_tags(
-    deployment_type,
-    extra_tags_array,
-    git_sha,
-    project,
-    project_version,
-    results_type,
-    step,
-    testcase_name,
+        deployment_type,
+        extra_tags_array,
+        git_sha,
+        project,
+        project_version,
+        results_type,
+        step,
+        testcase_name,
 ):
     common_broader_kv_tags = [
         {"project": project},

redisbench_admin/export/export.py

@@ -1,6 +1,9 @@
 import json
 import os
+
 import redis
+from redistimeseries.client import Client
+
 from redisbench_admin.export.common.common import split_tags_string
 from redisbench_admin.export.ftsb_redisearch.ftsb_redisearch_json_format import (
     ftsb_export_logic,
@@ -10,8 +13,6 @@
 )
 from redisbench_admin.utils.utils import retrieve_local_or_remote_input_json
 
-from redistimeseries.client import Client
-
 
 def export_command_logic(args):
     benchmark_files = args.benchmark_result_files

redisbench_admin/export/ftsb_redisearch/ftsb_redisearch_json_format.py

@@ -60,13 +60,13 @@ def extract_key_configs(benchmark_result):
 
 
 def ftsb_export_logic(
-    benchmark_result,
-    extra_tags_array,
-    filename,
-    included_steps,
-    results_type,
-    time_series_dict,
-    use_result,
+        benchmark_result,
+        extra_tags_array,
+        filename,
+        included_steps,
+        results_type,
+        time_series_dict,
+        use_result,
 ):
     key_result_steps = benchmark_result[results_type].keys()
     (

redisbench_admin/export/redis_benchmark/redis_benchmark_csv_format.py

@@ -52,7 +52,7 @@ def fill_tags_from_passed_array(extra_tags_array):
 
 
 def redis_benchmark_export_logic(
-    benchmark_result, extra_tags_array, results_type, time_series_dict, use_result
+        benchmark_result, extra_tags_array, results_type, time_series_dict, use_result
 ):
     ok = True
     start_time_ms = get_tag_fromextra_tags_array(extra_tags_array, "start_time_ms")

redisbench_admin/extract/extract.py

@@ -1,5 +1,5 @@
-import time
 import json
+import time
 
 import redis
 

redisbench_admin/run_remote/run_remote.py

@@ -3,17 +3,22 @@
 import logging
 import os
 import pathlib
+import re
 import sys
+import traceback
 
 import yaml
 from python_terraform import Terraform
 from redistimeseries.client import Client
 
+from redisbench_admin.run.redis_benchmark.redis_benchmark import prepareRedisBenchmarkCommand
+from redisbench_admin.run.run import redis_benchmark_ensure_min_version
 from redisbench_admin.utils.benchmark_config import (
     parseExporterMetricsDefinition,
     parseExporterTimeMetricDefinition,
     parseExporterTimeMetric,
 )
+from redisbench_admin.utils.local import prepareRedisGraphBenchmarkGoCommand
 from redisbench_admin.utils.redisgraph_benchmark_go import (
     spinUpRemoteRedis,
     setupRemoteBenchmark,
@@ -261,16 +266,59 @@ def run_remote_command_logic(args):
                             local_benchmark_output_filename
                         )
                     )
+
+                    benchmark_tool = None
+                    benchmark_min_tool_version = None
+                    benchmark_min_tool_version_major = None
+                    benchmark_min_tool_version_minor = None
+                    benchmark_min_tool_version_patch = None
+                    for entry in benchmark_config["clientconfig"]:
+                        benchmark_min_tool_version, benchmark_min_tool_version_major, benchmark_min_tool_version_minor, benchmark_min_tool_version_patch, benchmark_tool = extract_tool_info(
+                            benchmark_min_tool_version, benchmark_min_tool_version_major,
+                            benchmark_min_tool_version_minor, benchmark_min_tool_version_patch, benchmark_tool, entry)
+                    if benchmark_tool is not None:
+                        logging.info("Detected benchmark config tool {}".format(benchmark_tool))
+                    else:
+                        raise Exception("Unable to detect benchmark tool within 'clientconfig' section. Aborting...")
+
+                    if benchmark_tool not in args.allowed_tools.split(","):
+                        raise Exception(
+                            "Benchmark tool {} not in the allowed tools list [{}]. Aborting...".format(benchmark_tool,
+                                                                                                       args.allowed_tools))
+
+                    if benchmark_min_tool_version is not None and benchmark_tool == "redis-benchmark":
+                        redis_benchmark_ensure_min_version(benchmark_tool, benchmark_min_tool_version,
+                                                           benchmark_min_tool_version_major,
+                                                           benchmark_min_tool_version_minor,
+                                                           benchmark_min_tool_version_patch)
+
+                    for entry in benchmark_config["clientconfig"]:
+                        if 'parameters' in entry:
+                            if benchmark_tool == 'redis-benchmark':
+                                command = prepareRedisBenchmarkCommand(
+                                    "redis-benchmark",
+                                    server_private_ip,
+                                    server_plaintext_port,
+                                    entry
+                                )
+                            if benchmark_tool == 'redisgraph-benchmark-go':
+                                print(entry)
+                                command = prepareRedisGraphBenchmarkGoCommand(
+                                    "/tmp/redisgraph-benchmark-go",
+                                    server_private_ip,
+                                    server_plaintext_port,
+                                    entry,
+                                    remote_results_file,
+                                )
+
                     # run the benchmark
                     runRemoteBenchmark(
                         client_public_ip,
                         username,
                         private_key,
-                        server_private_ip,
-                        server_plaintext_port,
-                        benchmark_config,
                         remote_results_file,
                         local_benchmark_output_filename,
+                        command
                     )
 
                     # check KPIs
@@ -362,10 +410,32 @@ def run_remote_command_logic(args):
                         "Some unexpected exception was caught during remote work. Failing test...."
                     )
                     logging.critical(sys.exc_info()[0])
+                    print("-" * 60)
+                    traceback.print_exc(file=sys.stdout)
+                    print("-" * 60)
                 finally:
                     # tear-down
                     logging.info("Tearing down setup")
                     tf_output = tf.destroy()
                     logging.info("Tear-down completed")
 
     exit(return_code)
+
+
+def extract_tool_info(benchmark_min_tool_version, benchmark_min_tool_version_major, benchmark_min_tool_version_minor,
+                      benchmark_min_tool_version_patch, benchmark_tool, entry):
+    if 'tool' in entry:
+        benchmark_tool = entry['tool']
+    if 'min-tool-version' in entry:
+        benchmark_min_tool_version = entry['min-tool-version']
+        p = re.compile("(\d+)\.(\d+)\.(\d+)")
+        m = p.match(benchmark_min_tool_version)
+        if m is None:
+            logging.error(
+                "Unable to extract semversion from 'min-tool-version'. Will not enforce version")
+            benchmark_min_tool_version = None
+        else:
+            benchmark_min_tool_version_major = m.group(1)
+            benchmark_min_tool_version_minor = m.group(2)
+            benchmark_min_tool_version_patch = m.group(3)
+    return benchmark_min_tool_version, benchmark_min_tool_version_major, benchmark_min_tool_version_minor, benchmark_min_tool_version_patch, benchmark_tool

redisbench_admin/utils/benchmark_config.py

@@ -2,7 +2,7 @@
 
 
 def parseExporterMetricsDefinition(
-    benchmark_config: dict, configkey: str = "redistimeseries"
+        benchmark_config: dict, configkey: str = "redistimeseries"
 ):
     metrics = []
     if configkey in benchmark_config:
@@ -13,7 +13,7 @@ def parseExporterMetricsDefinition(
 
 
 def parseExporterTimeMetricDefinition(
-    benchmark_config: dict, configkey: str = "redistimeseries"
+        benchmark_config: dict, configkey: str = "redistimeseries"
 ):
     metricPath = None
     if "timemetric" in benchmark_config[configkey]:

redisbench_admin/utils/local.py

@@ -8,13 +8,13 @@
 import redis
 
 
-def checkDatasetLocalRequirements(benchmark_config, redis_tmp_dir, dirname = "."):
+def checkDatasetLocalRequirements(benchmark_config, redis_tmp_dir, dirname="."):
     for k in benchmark_config["dbconfig"]:
         if "dataset" in k:
             dataset = k["dataset"]
     if dataset is not None:
-        logging.info("Copying rdb {}/{} to {}/dump.rdb".format(dirname,dataset, redis_tmp_dir))
-        copyfile("{}/{}".format(dirname,dataset), "{}/dump.rdb".format(redis_tmp_dir))
+        logging.info("Copying rdb {}/{} to {}/dump.rdb".format(dirname, dataset, redis_tmp_dir))
+        copyfile("{}/{}".format(dirname, dataset), "{}/dump.rdb".format(redis_tmp_dir))
 
 
 def waitForConn(conn, retries=20, command="PING", shouldBe=True):
@@ -40,9 +40,9 @@ def waitForConn(conn, retries=20, command="PING", shouldBe=True):
 
 
 def spinUpLocalRedis(
-    benchmark_config,
-    port,
-    local_module_file,
+        benchmark_config,
+        port,
+        local_module_file,
         dirname=".",
 ):
     # copy the rdb to DB machine
@@ -53,7 +53,7 @@ def spinUpLocalRedis(
             temporary_dir
         )
     )
-    checkDatasetLocalRequirements(benchmark_config, temporary_dir,dirname)
+    checkDatasetLocalRequirements(benchmark_config, temporary_dir, dirname)
 
     # start redis-server
     command = [
@@ -90,9 +90,9 @@ def isProcessAlive(process):
 
 
 def getLocalRunFullFilename(
-    start_time_str,
-    github_branch,
-    test_name,
+        start_time_str,
+        github_branch,
+        test_name,
 ):
     benchmark_output_filename = (
         "{start_time_str}-{github_branch}-{test_name}.json".format(
@@ -105,12 +105,12 @@ def getLocalRunFullFilename(
 
 
 def prepareRedisGraphBenchmarkGoCommand(
-    executable_path: str,
-    server_private_ip: object,
-    server_plaintext_port: object,
-    benchmark_config: object,
-    results_file: object,
-) -> str:
+        executable_path: str,
+        server_private_ip: object,
+        server_plaintext_port: object,
+        benchmark_config: object,
+        results_file: object,
+):
     """
     Prepares redisgraph-benchmark-go command parameters
     :param server_private_ip:
@@ -122,7 +122,7 @@ def prepareRedisGraphBenchmarkGoCommand(
     queries_str = [executable_path]
     for k in benchmark_config["parameters"]:
         if "graph" in k:
-            queries_str.extend(["-graph-key", "{}".format(k["graph"])])
+            queries_str.extend(["-graph-key", "'{}'".format(k["graph"])])
         if "clients" in k:
             queries_str.extend(["-c", "{}".format(k["clients"])])
         if "requests" in k:
@@ -132,7 +132,7 @@ def prepareRedisGraphBenchmarkGoCommand(
         if "queries" in k:
             for kk in k["queries"]:
                 query = kk["q"]
-                queries_str.extend(["-query", "{}".format(query)])
+                queries_str.extend(["-query", "'{}'".format(query)])
                 if "ratio" in kk:
                     queries_str.extend(["-query-ratio", "{}".format(kk["ratio"])])
     queries_str.extend(["-h", "{}".format(server_private_ip)])