import json import os import time from cutecharts.charts import Pie from ovos_local_backend.database.metrics import JsonMetricDatabase from ovos_local_backend.database.settings import DeviceDatabase from ovos_local_backend.database.utterances import JsonUtteranceDatabase from ovos_local_backend.database.wakewords import JsonWakeWordDatabase from pywebio.input import actions from pywebio.output import put_text, popup, put_code, put_markdown, put_html, use_scope, put_image def device_select(back_handler=None): devices = {uuid: f"{device['name']}@{device['device_location']}" for uuid, device in DeviceDatabase().items()} buttons = [{'label': "All Devices", 'value': "all"}] + \ [{'label': d, 'value': uuid} for uuid, d in devices.items()] if back_handler: buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) if devices: uuid = actions(label="What device would you like to inspect?", buttons=buttons) if uuid == "main": metrics_menu(back_handler=back_handler) return else: if uuid == "all": uuid = None if uuid is not None: with use_scope("main_view", clear=True): put_markdown(f"\nDevice: {uuid}") metrics_menu(uuid=uuid, back_handler=back_handler) else: popup("No devices paired yet!") metrics_menu(back_handler=back_handler) def metrics_select(back_handler=None, uuid=None): buttons = [] db = JsonMetricDatabase() if not len(db): with use_scope("main_view", clear=True): put_text("No metrics uploaded yet!") metrics_menu(back_handler=back_handler, uuid=uuid) return for m in db: name = f"{m['metric_id']}-{m['metric_type']}" if uuid is not None and m["uuid"] != uuid: continue buttons.append({'label': name, 'value': m['metric_id']}) if back_handler: buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) opt = actions(label="Select a metric to inspect", buttons=buttons) if opt == "main": device_select(back_handler=back_handler) return # id == db_position + 1 with use_scope("main_view", clear=True): put_markdown("# Metadata") put_code(json.dumps(db[opt - 1], indent=4), "json") metrics_select(back_handler=back_handler, uuid=uuid) def metrics_menu(back_handler=None, uuid=None): with use_scope("logo", clear=True): img = open(f'{os.path.dirname(__file__)}/res/metrics.png', 'rb').read() put_image(img) buttons = [{'label': 'Metric Types', 'value': "types"}, {'label': 'Intents', 'value': "intents"}, {'label': 'FallbackSkill', 'value': "fallback"}, {'label': 'STT', 'value': "stt"}, {'label': 'TTS', 'value': "tts"}, {'label': 'Wake Words', 'value': "ww"}, {'label': 'Open Dataset', 'value': "opt-in"}] if uuid is not None: buttons.append({'label': 'Delete Device metrics', 'value': "delete_metrics"}) else: buttons.insert(1, {'label': 'Devices', 'value': "devices"}) buttons.append({'label': 'Inspect Devices', 'value': "metrics"}) buttons.append({'label': 'Delete ALL metrics', 'value': "delete_metrics"}) if back_handler: buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) opt = actions(label="What would you like to do?", buttons=buttons) if uuid is not None: m = DeviceMetricsReportGenerator(uuid) else: m = MetricsReportGenerator() if opt == "opt-in": with use_scope("main_view", clear=True): if uuid is None: md = f"""# Open Dataset Report Total Registered Devices: {len(DeviceDatabase())} Currently Opted-in: {len([d for d in DeviceDatabase() if d.opt_in])} Unique Devices seen: {m.total_devices}""" else: md = f"""Device: {uuid} # Open Dataset Report""" md += f""" Total Metrics submitted: {m.total_metrics} Total WakeWords submitted: {m.total_ww} Total Utterances submitted: {m.total_utt}""" put_markdown(md) put_html(m.optin_chart().render_notebook()) if opt == "devices": with use_scope("main_view", clear=True): md = f"""# Devices Report Total Devices: {m.total_devices} Total untracked: {len(m.untracked_devices)} Total active (estimate): {len(m.active_devices)} Total dormant (estimate): {len(m.dormant_devices)}""" put_markdown(md) put_html(m.device_chart().render_notebook()) if opt == "intents": with use_scope("main_view", clear=True): txt_estimate = max(m.total_intents + m.total_fallbacks - m.total_stt, 0) stt_estimate = max(m.total_intents + m.total_fallbacks - txt_estimate, 0) if uuid is not None: put_markdown(f"\nDevice: {uuid}") md = f"""# Intent Matches Report Total queries: {m.total_intents + m.total_fallbacks} Total text queries (estimate): {txt_estimate} Total speech queries (estimate): {stt_estimate} Total Matches: {m.total_intents}""" put_markdown(md) put_html(m.intent_type_chart().render_notebook()) if opt == "ww": bad = max(0, m.total_stt - m.total_ww) silents = max(0, m.total_stt - m.total_utt) with use_scope("main_view", clear=True): if uuid is not None: put_markdown(f"\nDevice: {uuid}\n\n") put_markdown(f"""Total WakeWord uploads: {m.total_ww} Total WakeWord detections (estimate): {m.total_stt} False Activations (estimate): {bad or silents} Silent Activations (estimate): {silents}""") put_html(m.ww_chart().render_notebook()) if opt == "stt": silents = max(0, m.total_stt - m.total_utt) with use_scope("main_view", clear=True): if uuid is not None: put_markdown(f"\nDevice: {uuid}\n\n") put_markdown(f"""Total Transcriptions: {m.total_stt} Total Recording uploads: {m.total_utt} Silent Activations (estimate): {silents}""") put_html(m.stt_type_chart().render_notebook()) if opt == "tts": with use_scope("main_view", clear=True): if uuid is not None: put_markdown(f"\nDevice: {uuid}") put_html(m.tts_type_chart().render_notebook()) if opt == "types": with use_scope("main_view", clear=True): if uuid is not None: put_markdown(f"\nDevice: {uuid}") put_markdown(f""" # Metrics Report Total Intents: {m.total_intents} Total Fallbacks: {m.total_fallbacks} Total Transcriptions: {m.total_stt} Total TTS: {m.total_tts} """) put_html(m.metrics_type_chart().render_notebook()) if opt == "fallback": with use_scope("main_view", clear=True): if uuid is not None: put_markdown(f"\nDevice: {uuid}") f = 0 if m.total_intents + m.total_fallbacks > 0: f = m.total_intents / (m.total_intents + m.total_fallbacks) put_markdown(f""" # Fallback Matches Report Total queries: {m.total_intents + m.total_fallbacks} Total Intents: {m.total_intents} Total Fallbacks: {m.total_fallbacks} Failure Percentage (estimate): {1 - f} """) put_html(m.fallback_type_chart().render_notebook()) if opt == "metrics": device_select(back_handler=back_handler) if opt == "delete_metrics": if uuid is not None: with use_scope("main_view", clear=True): put_markdown(f"\nDevice: {uuid}") with popup("Are you sure you want to delete the metrics database?"): put_text("this can not be undone, proceed with caution!") put_text("ALL metrics will be lost") opt = actions(label="Delete metrics database?", buttons=[{'label': "yes", 'value': True}, {'label': "no", 'value': False}]) if opt: os.remove(JsonMetricDatabase().db.path) with use_scope("main_view", clear=True): if back_handler: back_handler() else: metrics_menu(back_handler=back_handler, uuid=uuid) return if opt == "main": with use_scope("main_view", clear=True): if uuid is not None: device_select(back_handler=back_handler) elif back_handler: back_handler() return metrics_menu(back_handler=back_handler, uuid=uuid) class MetricsReportGenerator: def __init__(self): self.total_intents = 0 self.total_fallbacks = 0 self.total_stt = 0 self.total_tts = 0 self.total_ww = len(JsonWakeWordDatabase()) self.total_utt = len(JsonUtteranceDatabase()) self.total_devices = len(DeviceDatabase()) self.total_metrics = len(JsonMetricDatabase()) self.intents = {} self.fallbacks = {} self.ww = {} self.tts = {} self.stt = {} self.devices = {} self.load_metrics() @property def active_devices(self): thresh = time.time() - 7 * 24 * 60 * 60 return [uuid for uuid, ts in self.devices.items() if ts > thresh and uuid not in self.untracked_devices] @property def dormant_devices(self): return [uuid for uuid in self.devices.keys() if uuid not in self.untracked_devices and uuid not in self.active_devices] @property def untracked_devices(self): return [dev.uuid for dev in DeviceDatabase() if not dev.opt_in] def device_chart(self): chart = Pie("Devices") chart.set_options( labels=["active", "dormant", "untracked"], inner_radius=0, ) chart.add_series([len(self.active_devices), len(self.dormant_devices), len(self.untracked_devices)]) return chart def ww_chart(self): chart = Pie("Wake Words") labels = [] series = [] for ww, count in self.ww.items(): labels.append(ww) series.append(count) chart.set_options( labels=labels, inner_radius=0, ) chart.add_series(series) return chart def optin_chart(self): chart = Pie("Uploaded Data") chart.set_options( labels=["wake-words", "utterances", "metrics"], inner_radius=0, ) chart.add_series([self.total_ww, self.total_utt, self.total_metrics]) return chart def metrics_type_chart(self): chart = Pie("Metric Types") chart.set_options( labels=["intents", "fallbacks", "stt", "tts"], inner_radius=0, ) chart.add_series([self.total_intents, self.total_fallbacks, self.total_stt, self.total_tts]) return chart def intent_type_chart(self): chart = Pie("Intent Matches") chart.set_options( labels=list(self.intents.keys()), inner_radius=0, ) chart.add_series(list(self.intents.values())) return chart def fallback_type_chart(self): chart = Pie("Fallback Skills") chart.set_options( labels=list(self.fallbacks.keys()), inner_radius=0, ) chart.add_series(list(self.fallbacks.values())) return chart def tts_type_chart(self): chart = Pie("Text To Speech Engines") chart.set_options( labels=list(self.tts.keys()), inner_radius=0, ) chart.add_series(list(self.tts.values())) return chart def stt_type_chart(self): chart = Pie("Speech To Text Engines") chart.set_options( labels=list(self.stt.keys()), inner_radius=0, ) chart.add_series(list(self.stt.values())) return chart def reset_metrics(self): self.total_intents = 0 self.total_fallbacks = 0 self.total_stt = 0 self.total_tts = 0 self.total_ww = len(JsonWakeWordDatabase()) self.total_metrics = len(JsonMetricDatabase()) self.total_utt = len(JsonUtteranceDatabase()) self.total_devices = 0 self.intents = {} self.devices = {} self.fallbacks = {} self.tts = {} self.stt = {} self.ww = {} def load_metrics(self): self.reset_metrics() for m in JsonMetricDatabase(): if m["uuid"] not in self.devices: self.total_devices += 1 self._process_metric(m) for ww in JsonWakeWordDatabase(): if ww["meta"]["name"] not in self.ww: self.ww[ww["meta"]["name"]] = 0 else: self.ww[ww["meta"]["name"]] += 1 def _process_metric(self, m): if m["uuid"] not in self.devices or \ m["meta"]["time"] > self.devices[m["uuid"]]: self.devices[m["uuid"]] = m["meta"]["time"] if m["metric_type"] == "intent_service": self.total_intents += 1 k = f"{m['meta']['intent_type']}" if k not in self.intents: self.intents[k] = 0 self.intents[k] += 1 if m["metric_type"] == "fallback_handler": self.total_fallbacks += 1 k = f"{m['meta']['handler']}" if m['meta'].get("skill_id"): k = f"{m['meta']['skill_id']}:{m['meta']['handler']}" if k not in self.fallbacks: self.fallbacks[k] = 0 self.fallbacks[k] += 1 if m["metric_type"] == "stt": self.total_stt += 1 k = f"{m['meta']['stt']}" if k not in self.stt: self.stt[k] = 0 self.stt[k] += 1 if m["metric_type"] == "speech": self.total_tts += 1 k = f"{m['meta']['tts']}" if k not in self.tts: self.tts[k] = 0 self.tts[k] += 1 class DeviceMetricsReportGenerator(MetricsReportGenerator): def __init__(self, uuid): self.uuid = uuid super().__init__() def load_metrics(self): self.reset_metrics() self.total_ww = len([ww for ww in JsonWakeWordDatabase() if ww["uuid"] == self.uuid]) self.total_metrics = 0 self.total_utt = len([utt for utt in JsonUtteranceDatabase() if utt["uuid"] == self.uuid]) for m in JsonMetricDatabase(): if m["uuid"] != self.uuid: continue self._process_metric(m) self.total_metrics += 1 for ww in JsonWakeWordDatabase(): if ww["uuid"] != self.uuid: continue if ww["meta"]["name"] not in self.ww: self.ww[ww["meta"]["name"]] = 0 else: self.ww[ww["meta"]["name"]] += 1 if __name__ == "__main__": for ww in JsonWakeWordDatabase(): print(ww)