add system_details to SysOutput class

Former-commit-id: 217d13b

Author: pfliu-nlp

data/system_details/test.json

@@ -0,0 +1,4 @@
+{
+  "learning_rate": 0.0001,
+  "number_of_layers": 10
+}

explainaboard/analyzers/bar_chart.py

@@ -9,7 +9,7 @@
 mlogger.setLevel(logging.WARNING)
 
 
-def mp_format(data: list[dict]) -> dict:
+def mp_format(data) -> dict:
     """
     Adapt the format of data
     :param data:

explainaboard/explainaboard_main.py

@@ -161,6 +161,14 @@ def main():
         help="the directory of output files",
     )
 
+    parser.add_argument(
+        '--system_details',
+        type=str,
+        required=False,
+        default="system_details.json",
+        help="a json file to store detailed information for a system",
+    )
+
     args = parser.parse_args()
 
     dataset = args.dataset
@@ -181,6 +189,17 @@ def main():
     datasets_aggregation = args.datasets_aggregation
     languages_aggregation = args.languages_aggregation
 
+    system_details_path = args.system_details
+
+    # get system_details from input json file
+    system_details = None
+    if system_details_path is not None:
+        try:
+            with open(system_details_path) as fin:
+                system_details = json.load(fin)
+        except ValueError as e:
+            print('invalid json: %s' % e)
+
     # If reports have been specified, ExplainaBoard cli will perform analysis
     # over report files.
     if reports is not None:
@@ -329,6 +348,7 @@ def main():
         "task_name": task,
         "reload_stat": reload_stat,
         "conf_value": args.conf_value,
+        "system_details": system_details,
     }
 
     if metric_names is not None:

explainaboard/info.py

@@ -90,6 +90,9 @@ class SysOutputInfo:
     is_print_case: bool = True
     language: str = "en"
     conf_value: float = 0.05
+    system_details: dict = (
+        None  # TODO(Pengfei): we can define a schema using `dataclass` in the future
+    )
 
     # set later
     # code: str = None

explainaboard/tests/artifacts/test_system_details.json

@@ -0,0 +1,4 @@
+{
+  "learning_rate": 0.0001,
+  "number_of_layers": 10
+}

explainaboard/tests/test_system_details.py

@@ -0,0 +1,45 @@
+import json
+import os
+import pathlib
+import unittest
+
+from explainaboard import FileType, get_loader, get_processor, Source, TaskType
+
+artifacts_path = os.path.dirname(pathlib.Path(__file__)) + "/artifacts/"
+
+
+class TestSysDetails(unittest.TestCase):
+    def test_generate_system_analysis(self):
+        """TODO: should add harder tests"""
+
+        path_system_details = artifacts_path + "test_system_details.json"
+        path_data = artifacts_path + "sys_out1.tsv"
+
+        with open(path_system_details) as fin:
+            system_details = json.load(fin)
+
+        metadata = {
+            "task_name": TaskType.text_classification.value,
+            "metric_names": ["Accuracy"],
+            "system_details": system_details,
+        }
+
+        loader = get_loader(
+            TaskType.text_classification,
+            path_data,
+            Source.local_filesystem,
+            FileType.tsv,
+        )
+        data = list(loader.load())
+        processor = get_processor(TaskType.text_classification)
+
+        sys_info = processor.process(metadata, data)
+
+        # analysis.write_to_directory("./")
+        self.assertIsNotNone(
+            sys_info.system_details, {"learning_rate": 0.0001, "number_of_layers": 10}
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()