Skip to content

add multi metrics monitor for keras callback #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

add multi metrics monitor for keras callback #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c0a426c
add multi metrics monitor
Jul 5, 2018
6ea24f9
fix doc format
Jul 5, 2018
c7567f3
update for <python36
Jul 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions hyperdash/experiment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
import sys
import uuid
import threading

import numpy as np

from threading import Lock

from datetime import datetime
Expand Down Expand Up @@ -217,13 +220,13 @@ def __init__(self, exp):
def on_epoch_end(self, epoch, logs=None):
if not logs:
logs = {}
val_acc = logs.get("val_acc")
val_loss = logs.get("val_loss")
# for <python 36
for k, v in sorted(logs.items()):
if isinstance(v, (np.ndarray, np.generic)):
self._exp.metric(k, v.item())
else:
self._exp.metric(k, v)

if val_acc is not None:
self._exp.metric("val_acc", val_acc)
if val_loss is not None:
self._exp.metric("val_loss", val_loss)
cb = _KerasCallback(self._exp)
self._callbacks[KERAS] = cb
return cb
Expand Down
22 changes: 12 additions & 10 deletions tests/test_sdk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ def test_experiment(self):
exp.metric("accuracy", i*0.2)
time.sleep(0.1)
exp.end()

# Test params match what is expected
params_messages = []
for msg in server_sdk_messages:
Expand All @@ -355,7 +355,7 @@ def test_experiment(self):
},
"is_internal": True,
},
]
]
assert len(expect_params) == len(params_messages)
for i, message in enumerate(params_messages):
assert message == expect_params[i]
Expand All @@ -372,20 +372,20 @@ def test_experiment(self):
{"is_internal": False, "name": "accuracy", "value": 0},
{"is_internal": True, "name": "hd_iter_0", "value": 1},
{"is_internal": False, "name": "accuracy", "value": 0.2},
]
]
assert len(expect_metrics) == len(metrics_messages)
for i, message in enumerate(metrics_messages):
assert message["is_internal"] == expect_metrics[i]["is_internal"]
assert message["name"] == expect_metrics[i]["name"]
assert message["value"] == expect_metrics[i]["value"]

captured_out = faked_out.getvalue()
assert "error" not in captured_out

# Make sure correct API name / version headers are sent
assert server_sdk_headers[0][API_KEY_NAME] == API_NAME_EXPERIMENT
assert server_sdk_headers[0][VERSION_KEY_NAME] == get_hyperdash_version()

# Make sure logs were persisted
expect_logs = [
"{ batch size: 32 }",
Expand All @@ -410,10 +410,10 @@ def test_experiment_keras_callback(self):
with patch("sys.stdout", new=StringIO()) as faked_out:
exp = Experiment("MNIST")
keras_cb = exp.callbacks.keras
keras_cb.on_epoch_end(0, {"val_acc": 1, "val_loss": 2})
keras_cb.on_epoch_end(0, {"val_acc": 1, "val_loss": 2, "any_metrics": 10})
# Sleep 1 second due to client sampling
time.sleep(1)
keras_cb.on_epoch_end(1, {"val_acc": 3, "val_loss": 4})
keras_cb.on_epoch_end(1, {"val_acc": 3, "val_loss": 4, "any_metrics": 20})
exp.end()

# Test metrics match what is expected
Expand All @@ -423,8 +423,10 @@ def test_experiment_keras_callback(self):
if "name" in payload:
metrics_messages.append(payload)
expect_metrics = [
{"is_internal": False, "name": "any_metrics", "value": 10},
{"is_internal": False, "name": "val_acc", "value": 1},
{"is_internal": False, "name": "val_loss", "value": 2},
{"is_internal": False, "name": "any_metrics", "value": 20},
{"is_internal": False, "name": "val_acc", "value": 3},
{"is_internal": False, "name": "val_loss", "value": 4},
]
Expand All @@ -433,7 +435,7 @@ def test_experiment_keras_callback(self):
assert message["is_internal"] == expect_metrics[i]["is_internal"]
assert message["name"] == expect_metrics[i]["name"]
assert message["value"] == expect_metrics[i]["value"]

captured_out = faked_out.getvalue()
assert "error" not in captured_out

Expand All @@ -460,7 +462,7 @@ def test_experiment_handles_numpy_numbers(self):
exp.metric("test_metric_{}".format(name), num)
exp.param("test_param_{}".format(name), num)
exp.end()

# Test params match what is expected
params_messages = []
for msg in server_sdk_messages:
Expand Down Expand Up @@ -502,7 +504,7 @@ def test_experiment_handles_numpy_numbers(self):
assert message["is_internal"] == expected_metrics[i]["is_internal"]
assert message["name"] == expected_metrics[i]["name"]
assert message["value"] == expected_metrics[i]["value"]

def experiment_raises_exceptions(self):
exception_raised = True
expected_exception = "some_exception_b"
Expand Down