import json import os import time from cutecharts.charts import Pie, Bar, Scatter from ovos_backend_manager.configuration import CONFIGURATION, DB from pywebio.input import actions from pywebio.output import put_text, popup, put_code, put_markdown, put_html, use_scope, put_image chart_type = Pie def device_select(back_handler=None): devices = {device["uuid"]: f"{device['name']}@{device['device_location']}" for device in DB.list_devices()} buttons = [{'label': "All Devices", 'value': "all"}] + \ [{'label': d, 'value': uuid} for uuid, d in devices.items()] if back_handler: buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) if devices: uuid = actions(label="What device would you like to inspect?", buttons=buttons) if uuid == "main": metrics_menu(back_handler=back_handler) return else: if uuid == "all": uuid = None if uuid is not None: with use_scope("main_view", clear=True): put_markdown(f"\nDevice: {uuid}") metrics_menu(uuid=uuid, back_handler=back_handler) else: popup("No devices paired yet!") metrics_menu(back_handler=back_handler) def metrics_select(back_handler=None, uuid=None): buttons = [] metrics = DB.list_metrics() if not len(metrics): with use_scope("main_view", clear=True): put_text("No metrics uploaded yet!") metrics_menu(back_handler=back_handler, uuid=uuid) return for m in metrics: name = f"{m['metric_id']}-{m['metric_type']}" if uuid is not None and m["uuid"] != uuid: continue buttons.append({'label': name, 'value': m['metric_id']}) if back_handler: buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) metric_id = actions(label="Select a metric to inspect", buttons=buttons) if metric_id == "main": device_select(back_handler=back_handler) return with use_scope("main_view", clear=True): put_markdown("# Metadata") put_code(json.dumps(metric_id, indent=4), "json") metrics_select(back_handler=back_handler, uuid=uuid) def _plot_metrics(uuid, selected_metric="types"): if uuid is not None: m = DeviceMetricsReportGenerator(uuid) else: m = MetricsReportGenerator() with use_scope("main_view", clear=True): if uuid is not None: put_markdown(f"\nDevice: {uuid}") if selected_metric == "timings": put_html(m.timings_chart().render_notebook()) elif selected_metric == "stt": silents = max(0, m.total_stt - m.total_utt) put_markdown(f"""Total Transcriptions: {m.total_stt} Total Recording uploads: {m.total_utt} Silent Activations (estimate): {silents}""") if chart_type == Pie: put_html(m.stt_pie_chart().render_notebook()) else: put_html(m.stt_bar_chart().render_notebook()) elif selected_metric == "devices": md = f"""# Devices Report Total Devices: {m.total_devices} Total untracked: {len(m.untracked_devices)} Total active (estimate): {len(m.active_devices)} Total dormant (estimate): {len(m.dormant_devices)}""" put_markdown(md) if chart_type == Pie: put_html(m.devices_pie_chart().render_notebook()) else: put_html(m.devices_bar_chart().render_notebook()) elif selected_metric == "intents": txt_estimate = max(m.total_intents + m.total_fallbacks - m.total_stt, 0) stt_estimate = max(m.total_intents + m.total_fallbacks - txt_estimate, 0) md = f"""# Intent Matches Report Total queries: {m.total_intents + m.total_fallbacks} Total text queries (estimate): {txt_estimate} Total speech queries (estimate): {stt_estimate} Total Matches: {m.total_intents}""" put_markdown(md) if chart_type == Bar: put_html(m.intents_bar_chart().render_notebook()) else: put_html(m.intents_pie_chart().render_notebook()) elif selected_metric == "ww": bad = max(0, m.total_stt - m.total_ww) silents = max(0, m.total_stt - m.total_utt) put_markdown(f"""Total WakeWord uploads: {m.total_ww} Total WakeWord detections (estimate): {m.total_stt} False Activations (estimate): {bad or silents} Silent Activations (estimate): {silents}""") if chart_type == Pie: put_html(m.ww_pie_chart().render_notebook()) else: put_html(m.ww_bar_chart().render_notebook()) elif selected_metric == "tts": if chart_type == Pie: put_html(m.tts_pie_chart().render_notebook()) else: put_html(m.tts_bar_chart().render_notebook()) elif selected_metric == "types": put_markdown(f""" # Metrics Report Total Intents: {m.total_intents} Total Fallbacks: {m.total_fallbacks} Total Transcriptions: {m.total_stt} Total TTS: {m.total_tts} """) if chart_type == Pie: put_html(m.metrics_type_pie_chart().render_notebook()) else: put_html(m.metrics_type_bar_chart().render_notebook()) elif selected_metric == "fallback": f = 0 if m.total_intents + m.total_fallbacks > 0: f = m.total_intents / (m.total_intents + m.total_fallbacks) put_markdown(f""" # Fallback Matches Report Total queries: {m.total_intents + m.total_fallbacks} Total Intents: {m.total_intents} Total Fallbacks: {m.total_fallbacks} Failure Percentage (estimate): {1 - f} """) if chart_type == Pie: put_html(m.fallback_pie_chart().render_notebook()) else: put_html(m.fallback_bar_chart().render_notebook()) elif selected_metric == "opt-in": md = "" if uuid is None: md = f"""# Open Dataset Report Total Registered Devices: {len(DB.list_devices())} Currently Opted-in: {len([d for d in DB.list_devices() if d["opt_in"]])} Unique Devices seen: {m.total_devices}""" # Open Dataset Report""" md += f""" Total Metrics submitted: {m.total_metrics} Total WakeWords submitted: {m.total_ww} Total Utterances submitted: {m.total_utt}""" put_markdown(md) if chart_type == Pie: put_html(m.dataset_pie_chart().render_notebook()) else: put_html(m.dataset_bar_chart().render_notebook()) def metrics_menu(back_handler=None, uuid=None, selected_metric="types"): global chart_type with use_scope("logo", clear=True): img = open(f'{os.path.dirname(__file__)}/res/metrics.png', 'rb').read() put_image(img) _plot_metrics(uuid, selected_metric) buttons = [{'label': 'Timings', 'value': "timings"}, {'label': 'Metric Types', 'value': "types"}, {'label': 'Intents', 'value': "intents"}, {'label': 'FallbackSkill', 'value': "fallback"}, {'label': 'STT', 'value': "stt"}, {'label': 'TTS', 'value': "tts"}, {'label': 'Wake Words', 'value': "ww"}, {'label': 'Open Dataset', 'value': "opt-in"}] if chart_type == Pie: buttons.append({'label': 'Bar style graphs', 'value': "chart"}) elif chart_type == Bar: buttons.append({'label': 'Pie style graphs', 'value': "chart"}) if uuid is not None: buttons.append({'label': 'Delete Device metrics', 'value': "delete_metrics"}) else: buttons.insert(1, {'label': 'Devices', 'value': "devices"}) buttons.append({'label': 'Inspect Devices', 'value': "metrics"}) buttons.append({'label': 'Delete ALL metrics', 'value': "delete_metrics"}) if back_handler: buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) opt = actions(label="What would you like to do?", buttons=buttons) if opt == "chart": if chart_type == Pie: chart_type = Bar else: chart_type = Pie elif opt in ["devices", "intents", "stt", "ww", "tts", "types", "fallback", "opt-in", "timings"]: selected_metric = opt elif opt == "metrics": device_select(back_handler=back_handler) elif opt == "delete_metrics": if uuid is not None: with use_scope("main_view", clear=True): put_markdown(f"\nDevice: {uuid}") with popup("Are you sure you want to delete the metrics database?"): put_text("this can not be undone, proceed with caution!") put_text("ALL metrics will be lost") opt = actions(label="Delete metrics database?", buttons=[{'label': "yes", 'value': True}, {'label': "no", 'value': False}]) if opt: for m in DB.list_metrics(): DB.delete_metric(m["metric_id"]) with use_scope("main_view", clear=True): if back_handler: back_handler() else: metrics_menu(back_handler=back_handler, uuid=uuid, selected_metric=selected_metric) return elif opt == "main": with use_scope("main_view", clear=True): if uuid is not None: device_select(back_handler=back_handler) elif back_handler: back_handler() return metrics_menu(back_handler=back_handler, uuid=uuid, selected_metric=selected_metric) class MetricsReportGenerator: def __init__(self): self.total_intents = 0 self.total_fallbacks = 0 self.total_stt = 0 self.total_tts = 0 self.total_ww = len(DB.list_ww_recordings()) self.total_utt = len(DB.list_stt_recordings()) self.total_devices = len(DB.list_devices()) self.total_metrics = len(DB.list_metrics()) self.intents = {} self.fallbacks = {} self.ww = {} self.tts = {} self.stt = {} self.devices = {} self.stt_timings = [] self.tts_timings = [] self.intent_timings = [] self.fallback_timings = [] self.device_timings = [] self.load_metrics() def reset_metrics(self): self.total_intents = 0 self.total_fallbacks = 0 self.total_stt = 0 self.total_tts = 0 self.total_ww = len(DB.list_ww_recordings()) self.total_utt = len(DB.list_stt_recordings()) self.total_devices = 0 self.total_metrics = len(DB.list_metrics()) self.intents = {} self.devices = {} self.fallbacks = {} self.tts = {} self.stt = {} self.ww = {} self.stt_timings = [] self.tts_timings = [] self.intent_timings = [] self.fallback_timings = [] self.device_timings = [] def load_metrics(self): self.reset_metrics() for m in DB.list_metrics(): if m["uuid"] not in self.devices: self.total_devices += 1 self._process_metric(m) for ww in DB.list_ww_recordings(): if ww["meta"]["name"] not in self.ww: self.ww[ww["meta"]["name"]] = 0 else: self.ww[ww["meta"]["name"]] += 1 @property def active_devices(self): thresh = time.time() - 7 * 24 * 60 * 60 return [uuid for uuid, ts in self.devices.items() if ts > thresh and uuid not in self.untracked_devices] @property def dormant_devices(self): return [uuid for uuid in self.devices.keys() if uuid not in self.untracked_devices and uuid not in self.active_devices] @property def untracked_devices(self): return [dev["uuid"] for dev in DB.list_devices() if not dev["opt_in"]] # cute charts def timings_chart(self): chart = Scatter("Execution Time") chart.set_options(y_tick_count=8, is_show_line=True, x_label="Unix Time", y_label="Seconds") chart.add_series( "STT", [(z[0], z[1]) for z in self.stt_timings] ) chart.add_series( "TTS", [(z[0], z[1]) for z in self.tts_timings] ) chart.add_series( "Intent Matching", [(z[0], z[1]) for z in self.intent_timings] ) chart.add_series( "Fallback Handling", [(z[0], z[1]) for z in self.fallback_timings] ) return chart def devices_pie_chart(self): chart = Pie("Devices") chart.set_options( labels=["active", "dormant", "untracked"], inner_radius=0, ) chart.add_series([len(self.active_devices), len(self.dormant_devices), len(self.untracked_devices)]) return chart def devices_bar_chart(self): chart = Bar("Devices") chart.set_options( labels=["active", "dormant", "untracked"], x_label="Status", y_label="Number" ) chart.add_series("Count", [len(self.active_devices), len(self.dormant_devices), len(self.untracked_devices)]) return chart def ww_bar_chart(self): chart = Bar("Wake Words") labels = [] series = [] for ww, count in self.ww.items(): labels.append(ww) series.append(count) chart.set_options( labels=labels, x_label="Wake Word", y_label="# Submitted" ) chart.add_series("Count", series) return chart def ww_pie_chart(self): chart = Pie("Wake Words") labels = [] series = [] for ww, count in self.ww.items(): labels.append(ww) series.append(count) chart.set_options( labels=labels, inner_radius=0, ) chart.add_series(series) return chart def dataset_pie_chart(self): chart = Pie("Uploaded Data") chart.set_options( labels=["wake-words", "utterances", "metrics"], inner_radius=0, ) chart.add_series([self.total_ww, self.total_utt, self.total_metrics]) return chart def dataset_bar_chart(self): chart = Bar("Uploaded Data") chart.set_options( labels=["wake-words", "utterances", "metrics"], x_label="Data Type", y_label="# Submitted" ) chart.add_series("Count", [self.total_ww, self.total_utt, self.total_metrics]) return chart def metrics_type_bar_chart(self): chart = Bar("Metric Types") chart.set_options( labels=["intents", "fallbacks", "stt", "tts"], x_label="Metric Type", y_label="# Submitted" ) chart.add_series("Number", [self.total_intents, self.total_fallbacks, self.total_stt, self.total_tts]) return chart def metrics_type_pie_chart(self): chart = Pie("Metric Types") chart.set_options( labels=["intents", "fallbacks", "stt", "tts"], inner_radius=0, ) chart.add_series([self.total_intents, self.total_fallbacks, self.total_stt, self.total_tts]) return chart def intents_bar_chart(self): chart = Bar("Intent Matches") chart.set_options(labels=list(self.intents.keys()), x_label="Intent Name", y_label="Times Triggered") chart.add_series("Count", list(self.intents.values())) return chart def intents_pie_chart(self): chart = Pie("Intent Matches") chart.set_options( labels=list(self.intents.keys()), inner_radius=0, ) chart.add_series(list(self.intents.values())) return chart def fallback_bar_chart(self): chart = Bar("Fallback Skills") chart.set_options( labels=list(self.fallbacks.keys()), x_label="Fallback Handler", y_label="Times Triggered" ) chart.add_series("Count", list(self.fallbacks.values())) return chart def fallback_pie_chart(self): chart = Pie("Fallback Skills") chart.set_options( labels=list(self.fallbacks.keys()), inner_radius=0, ) chart.add_series(list(self.fallbacks.values())) return chart def tts_bar_chart(self): chart = Bar("Text To Speech Engines") chart.set_options( labels=list(self.tts.keys()), x_label="Engine", y_label="Times Triggered" ) chart.add_series("Count", list(self.tts.values())) return chart def tts_pie_chart(self): chart = Pie("Text To Speech Engines") chart.set_options( labels=list(self.tts.keys()), inner_radius=0, ) chart.add_series(list(self.tts.values())) return chart def stt_bar_chart(self): chart = Bar("Speech To Text Engines") chart.set_options( labels=list(self.stt.keys()), x_label="Engine", y_label="Times Triggered" ) chart.add_series("Count", list(self.stt.values())) return chart def stt_pie_chart(self): chart = Pie("Speech To Text Engines") chart.set_options( labels=list(self.stt.keys()), inner_radius=0, ) chart.add_series(list(self.stt.values())) return chart def _process_metric(self, m): start = m["meta"]["start_time"] end = m["meta"]["time"] duration = end - start if m["uuid"] not in self.devices or \ m["meta"]["time"] > self.devices[m["uuid"]]: self.devices[m["uuid"]] = m["meta"]["time"] if m["metric_type"] == "intent_service": label = m["meta"]["intent_type"] self.intent_timings.append((start, duration, label)) self.total_intents += 1 k = f"{m['meta']['intent_type']}" if k not in self.intents: self.intents[k] = 0 self.intents[k] += 1 if m["metric_type"] == "fallback_handler": self.total_fallbacks += 1 k = f"{m['meta']['handler']}" if m['meta'].get("skill_id"): k = f"{m['meta']['skill_id']}:{m['meta']['handler']}" if k not in self.fallbacks: self.fallbacks[k] = 0 self.fallbacks[k] += 1 label = k self.fallback_timings.append((start, duration, label)) if m["metric_type"] == "stt": label = m["meta"]["transcription"] self.stt_timings.append((start, duration, label)) self.total_stt += 1 k = f"{m['meta']['stt']}" if k not in self.stt: self.stt[k] = 0 self.stt[k] += 1 if m["metric_type"] == "speech": label = m["meta"]["utterance"] self.tts_timings.append((start, duration, label)) self.total_tts += 1 k = f"{m['meta']['tts']}" if k not in self.tts: self.tts[k] = 0 self.tts[k] += 1 self.device_timings.append((start, duration, m["uuid"])) # sort by timestamp self.device_timings = sorted(self.device_timings, key=lambda k: k[0], reverse=True) self.stt_timings = sorted(self.stt_timings, key=lambda k: k[0], reverse=True) self.tts_timings = sorted(self.tts_timings, key=lambda k: k[0], reverse=True) self.intent_timings = sorted(self.intent_timings, key=lambda k: k[0], reverse=True) self.fallback_timings = sorted(self.fallback_timings, key=lambda k: k[0], reverse=True) class DeviceMetricsReportGenerator(MetricsReportGenerator): def __init__(self, uuid): self.uuid = uuid super().__init__() def load_metrics(self): self.reset_metrics() self.total_ww = len([ww for ww in DB.list_ww_recordings() if ww["uuid"] == self.uuid]) self.total_metrics = 0 self.total_utt = len([utt for utt in DB.list_stt_recordings() if utt["uuid"] == self.uuid]) for m in DB.list_metrics(): if m["uuid"] != self.uuid: continue self._process_metric(m) self.total_metrics += 1 for ww in DB.list_ww_recordings(): if ww["uuid"] != self.uuid: continue if ww["meta"]["name"] not in self.ww: self.ww[ww["meta"]["name"]] = 0 else: self.ww[ww["meta"]["name"]] += 1 if __name__ == "__main__": for ww in DB.list_ww_recordings(): print(ww)