simcop2387/ovos-backend-manager
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

initial migration

Browse source
This commit is contained in:
JarbasAi 2023-04-08 00:50:24 +01:00
parent 31d8fdd3b1
commit 805cf3baf9
12 changed files with 370 additions and 525 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
helper/import_precise_community.py
View file

@ -1,7 +1,7 @@
import os import os
import random import random
from ovos_local_backend.configuration import CONFIGURATION from ovos_backend_manager.configuration import CONFIGURATION
from ovos_local_backend.database.metrics import JsonMetricDatabase, Metric from ovos_local_backend.database.metrics import JsonMetricDatabase, Metric
from ovos_local_backend.database.wakewords import JsonWakeWordDatabase, WakeWordRecording from ovos_local_backend.database.wakewords import JsonWakeWordDatabase, WakeWordRecording
from os.path import dirname from os.path import dirname

7
ovos_backend_manager/app.py
View file

@ -3,7 +3,7 @@ import os
import requests import requests
from flask import Flask, request from flask import Flask, request
from oauthlib.oauth2 import WebApplicationClient from oauthlib.oauth2 import WebApplicationClient
from ovos_local_backend.database.oauth import OAuthTokenDatabase, OAuthApplicationDatabase from ovos_backend_manager.configuration import DB
from pywebio.platform.flask import webio_view from pywebio.platform.flask import webio_view
from ovos_backend_manager.menu import start from ovos_backend_manager.menu import start
@ -23,7 +23,7 @@ def oauth_callback(oauth_id):
params = dict(request.args) params = dict(request.args)
code = params["code"] code = params["code"]
data = OAuthApplicationDatabase()[oauth_id] data = DB.get_oauth_app(oauth_id)
client_id = data["client_id"] client_id = data["client_id"]
client_secret = data["client_secret"] client_secret = data["client_secret"]
token_endpoint = data["token_endpoint"] token_endpoint = data["token_endpoint"]
@ -43,8 +43,7 @@ def oauth_callback(oauth_id):
auth=(client_id, client_secret), auth=(client_id, client_secret),
).json() ).json()
with OAuthTokenDatabase() as db: DB.add_oauth_token(oauth_id, token_response)
db.add_token(oauth_id, token_response)
return params return params

7
ovos_backend_manager/backend.py
View file

@ -1,8 +1,8 @@
import json import json
import os import os
from ovos_local_backend.configuration import CONFIGURATION from ovos_backend_manager.configuration import CONFIGURATION
from ovos_local_backend.utils.geolocate import get_location_config from ovos_backend_client.api import GeolocationApi
from pywebio.input import textarea, select, actions from pywebio.input import textarea, select, actions
from pywebio.output import put_table, put_markdown, popup, put_code, put_image, use_scope from pywebio.output import put_table, put_markdown, popup, put_code, put_image, use_scope
@ -18,7 +18,6 @@ def backend_menu(back_handler=None):
['Device Authentication enabled', not CONFIGURATION["skip_auth"]], ['Device Authentication enabled', not CONFIGURATION["skip_auth"]],
['Location override enabled', CONFIGURATION["override_location"]], ['Location override enabled', CONFIGURATION["override_location"]],
['IP Geolocation enabled', CONFIGURATION["geolocate"]], ['IP Geolocation enabled', CONFIGURATION["geolocate"]],
['Selene Proxy enabled', CONFIGURATION["selene"]["enabled"]],
['Default TTS', CONFIGURATION["default_tts"]], ['Default TTS', CONFIGURATION["default_tts"]],
['Default Wake Word', CONFIGURATION["default_ww"]], ['Default Wake Word', CONFIGURATION["default_ww"]],
['Default date format', CONFIGURATION["date_format"]], ['Default date format', CONFIGURATION["date_format"]],
@ -76,7 +75,7 @@ def backend_menu(back_handler=None):
loc = textarea("Enter an address", loc = textarea("Enter an address",
placeholder="Anywhere street Any city Nº234", placeholder="Anywhere street Any city Nº234",
required=True) required=True)
data = get_location_config(loc) data = GeolocationApi().get_geolocation(loc)
CONFIGURATION["default_location"] = data CONFIGURATION["default_location"] = data
with popup(f"Default location set to: {loc}"): with popup(f"Default location set to: {loc}"):
put_code(json.dumps(data, ensure_ascii=True, indent=2), "json") put_code(json.dumps(data, ensure_ascii=True, indent=2), "json")

215
ovos_backend_manager/configuration.py Normal file
View file

@ -0,0 +1,215 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from os.path import exists, expanduser
from json_database import JsonConfigXDG
from ovos_utils.log import LOG
from ovos_backend_client.api import DatabaseApi
DEFAULT_CONFIG = {
"lang": "en-us", # default language
"database": "sqlite:///ovos_backend.db",
"stt": {
"module": "ovos-stt-plugin-server",
"ovos-stt-plugin-server": {
"url": "https://stt.openvoiceos.com/stt"
}
},
"backend_port": 6712,
"admin_key": "", # To enable simply set this string to something
"skip_auth": False, # you almost certainly do not want this, only for atypical use cases such as ovos-qubes
"default_location": {
"city": {
"code": "Lawrence",
"name": "Lawrence",
"state": {
"code": "KS",
"name": "Kansas",
"country": {
"code": "US",
"name": "United States"
}
}
},
"coordinate": {
"latitude": 38.971669,
"longitude": -95.23525
},
"timezone": {
"code": "America/Chicago",
"name": "Central Standard Time",
"dstOffset": 3600000,
"offset": -21600000
}
},
"default_ww": "hey_mycroft", # needs to be present below
"ww_configs": { # these can be exposed in a web UI for selection
"android": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/android.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"computer": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/computer.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"hey_chatterbox": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_chatterbox.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"hey_firefox": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_firefox.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"hey_k9": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_k9.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"hey_kit": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_kit.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"hey_moxie": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_moxie.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"hey_mycroft": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_mycroft.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"hey_scout": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_scout.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"marvin": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/marvin.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"o_sauro": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/o_sauro.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"sheila": {"module": "ovos-ww-plugin-precise-lite",
"model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/sheila.tflite",
"expected_duration": 3,
"trigger_level": 3,
"sensitivity": 0.5
},
"hey_jarvis": {"module": "ovos-ww-plugin-vosk",
"rule": "fuzzy",
"samples": [
"hay jarvis",
"hey jarvis",
"hay jarbis",
"hey jarbis"
]
},
"christopher": {"module": "ovos-ww-plugin-vosk",
"rule": "fuzzy",
"samples": [
"christopher"
]
},
"hey_ezra": {"module": "ovos-ww-plugin-vosk",
"rule": "fuzzy",
"samples": [
"hay ezra",
"hey ezra"
]
},
"hey_ziggy": {"module": "ovos-ww-plugin-vosk",
"rule": "fuzzy",
"samples": [
"hey ziggy",
"hay ziggy"
]
},
"hey_neon": {"module": "ovos-ww-plugin-vosk",
"rule": "fuzzy",
"samples": [
"hey neon",
"hay neon"
]
}
},
"default_tts": "American Male", # needs to be present below
"tts_configs": { # these can be exposed in a web UI for selection
"American Male": {"module": "ovos-tts-plugin-mimic2", "voice": "kusal"},
"British Male": {"module": "ovos-tts-plugin-mimic", "voice": "ap"}
},
"date_format": "DMY",
"system_unit": "metric",
"time_format": "full",
"geolocate": True,
"override_location": False,
"api_version": "v1",
"data_path": expanduser("~"),
"record_utterances": False,
"record_wakewords": False,
"microservices": {
# if query fail, attempt to use free ovos services
"ovos_fallback": True,
# backend can be auto/local/ovos
# auto == attempt local -> ovos
"wolfram_provider": "auto",
"weather_provider": "auto",
# auto == OpenStreetMap default
# valid - osm/arcgis/geocode_farm
"geolocation_provider": "auto",
# secret keys
"wolfram_key": "",
"owm_key": ""
},
"email": {
"username": None,
"password": None
}
}
CONFIGURATION = JsonConfigXDG("ovos_backend")
if not exists(CONFIGURATION.path):
CONFIGURATION.merge(DEFAULT_CONFIG, skip_empty=False)
CONFIGURATION.store()
LOG.info(f"Saved default configuration: {CONFIGURATION.path}")
else:
# set any new default values since file creation
for k, v in DEFAULT_CONFIG.items():
if k not in CONFIGURATION:
CONFIGURATION[k] = v
LOG.info(f"Loaded configuration: {CONFIGURATION.path}")
DB = DatabaseApi(CONFIGURATION["admin_key"])

149
ovos_backend_manager/datasets.py
View file

@ -3,28 +3,25 @@ import os
import time import time
from base64 import b64encode from base64 import b64encode
from ovos_local_backend.configuration import CONFIGURATION
from ovos_local_backend.database.settings import DeviceDatabase
from ovos_local_backend.database.utterances import JsonUtteranceDatabase
from ovos_local_backend.database.wakewords import JsonWakeWordDatabase
from pywebio.input import actions, file_upload, input_group, textarea, select from pywebio.input import actions, file_upload, input_group, textarea, select
from pywebio.output import put_text, put_code, use_scope, put_markdown, popup, put_image, put_file, put_html, \ from pywebio.output import put_text, put_code, use_scope, put_markdown, popup, put_image, put_file, put_html, \
put_buttons, put_table put_buttons, put_table
from ovos_backend_manager.configuration import CONFIGURATION, DB
def _render_ww(idx, db=None):
db = db or JsonWakeWordDatabase() def _render_ww(rec_id):
def on_tag(bt): def on_tag(bt):
data["tag"] = bt data["tag"] = bt
db[idx]["tag"] = bt DB.update_ww_recording(rec_id, tag=bt)
db.commit() _render_ww(rec_id)
_render_ww(idx, db)
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
data = db[idx] # id == db_position + 1 data = DB.get_ww_recording(rec_id)
data["tag"] = data.get("tag") or "untagged" data["tag"] = data.get("tag") or "untagged"
# TODO - get binary_data directly
if os.path.isfile(data["path"]): if os.path.isfile(data["path"]):
content = open(data["path"], 'rb').read() content = open(data["path"], 'rb').read()
html = f""" html = f"""
@ -50,72 +47,73 @@ def _render_ww(idx, db=None):
def ww_select(back_handler=None, uuid=None, ww=None): def ww_select(back_handler=None, uuid=None, ww=None):
buttons = [] buttons = []
db = JsonWakeWordDatabase() if not len(DB.list_ww_recordings()):
if not len(db):
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_text("No wake words uploaded yet!") put_text("No wake words uploaded yet!")
datasets_menu(back_handler=back_handler) datasets_menu(back_handler=back_handler)
return return
for m in db: for m in DB.list_ww_recordings():
if uuid is not None and m["uuid"] != uuid: if uuid is not None and m["uuid"] != uuid:
continue continue
if ww is not None and m["transcription"] != ww: if ww is not None and m["transcription"] != ww:
continue continue
name = f"{m['wakeword_id']}-{m['transcription']}" name = f"{m['recording_id']}-{m['transcription']}"
buttons.append({'label': name, 'value': m['wakeword_id']}) buttons.append({'label': name, 'value': m['recording_id']})
if len(buttons) == 0: if len(buttons) == 0:
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_text("No wake words uploaded from this device yet!") put_text("No wake words uploaded from this device yet!")
opt = "main"
else:
if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
opt = actions(label="Select a WakeWord recording",
buttons=buttons)
if opt == "main":
ww_menu(back_handler=back_handler) ww_menu(back_handler=back_handler)
return return
_render_ww(opt - 1, db) elif back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
ww_select(back_handler=back_handler, ww=ww, uuid=uuid) rec_id = actions(label="Select a WakeWord recording", buttons=buttons)
if rec_id == "main": # go back
ww_menu(back_handler=back_handler)
else:
_render_ww(rec_id)
ww_select(back_handler=back_handler, ww=ww, uuid=uuid)
def utt_select(back_handler=None, uuid=None, utt=None): def utt_select(back_handler=None, uuid=None, utt=None):
buttons = [] buttons = []
db = JsonUtteranceDatabase() if not len(DB.list_stt_recordings()):
if not len(db):
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_text("No utterances uploaded yet!") put_text("No utterances uploaded yet!")
datasets_menu(back_handler=back_handler) datasets_menu(back_handler=back_handler)
return return
for m in db: for m in DB.list_stt_recordings():
if uuid is not None and m["uuid"] != uuid: if uuid is not None and m["uuid"] != uuid:
continue continue
if utt is not None and m["transcription"] != utt: if utt is not None and m["transcription"] != utt:
continue continue
name = f"{m['utterance_id']}-{m['transcription']}" name = f"{m['recording_id']}-{m['transcription']}"
buttons.append({'label': name, 'value': m['utterance_id']}) buttons.append({'label': name, 'value': m['recording_id']})
if len(buttons) == 0: if len(buttons) == 0:
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_text("No utterances uploaded from this device yet!") put_text("No utterances uploaded from this device yet!")
opt = "main" utt_menu(back_handler=back_handler)
return
else: else:
if back_handler: if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
opt = actions(label="Select a Utterance recording", rec_id = actions(label="Select a Utterance recording",
buttons=buttons) buttons=buttons)
if opt == "main": if rec_id == "main":
utt_menu(back_handler=back_handler) utt_menu(back_handler=back_handler)
return return
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
data = db[opt - 1] # id == db_position + 1 # opt is recording_id
data = DB.get_stt_recording(rec_id)
put_code(json.dumps(data, indent=4), "json") put_code(json.dumps(data, indent=4), "json")
# TODO - get binary data from api
if os.path.isfile(data["path"]): if os.path.isfile(data["path"]):
content = open(data["path"], 'rb').read() content = open(data["path"], 'rb').read()
html = f"""<audio controls src="data:audio/x-wav;base64,{b64encode(content).decode('ascii')}" />""" html = f"""<audio controls src="data:audio/x-wav;base64,{b64encode(content).decode('ascii')}" />"""
@ -128,8 +126,8 @@ def utt_select(back_handler=None, uuid=None, utt=None):
def device_select(back_handler=None, ww=True): def device_select(back_handler=None, ww=True):
devices = {uuid: f"{device['name']}@{device['device_location']}" devices = {device["uuid"]: f"{device['name']}@{device['device_location']}"
for uuid, device in DeviceDatabase().items()} for device in DB.list_devices()}
buttons = [{'label': "All Devices", 'value': "all"}, buttons = [{'label': "All Devices", 'value': "all"},
{'label': "Unknown Devices", 'value': "AnonDevice"}] + \ {'label': "Unknown Devices", 'value': "AnonDevice"}] + \
[{'label': d, 'value': uuid} for uuid, d in devices.items()] [{'label': d, 'value': uuid} for uuid, d in devices.items()]
@ -159,7 +157,7 @@ def device_select(back_handler=None, ww=True):
def ww_opts(back_handler=None, uuid=None): def ww_opts(back_handler=None, uuid=None):
wws = list(set([ww["transcription"] for ww in JsonWakeWordDatabase()])) wws = list(set([ww["transcription"] for ww in DB.list_ww_recordings()]))
buttons = [{'label': "All Wake Words", 'value': "all"}] + \ buttons = [{'label': "All Wake Words", 'value': "all"}] + \
[{'label': ww, 'value': ww} for ww in wws] [{'label': ww, 'value': ww} for ww in wws]
if back_handler: if back_handler:
@ -180,7 +178,7 @@ def ww_opts(back_handler=None, uuid=None):
def utt_opts(back_handler=None, uuid=None): def utt_opts(back_handler=None, uuid=None):
utts = list(set([ww["transcription"] for ww in JsonUtteranceDatabase()])) utts = list(set([ww["transcription"] for ww in DB.list_stt_recordings()]))
buttons = [{'label': "All Utterances", 'value': "all"}] + \ buttons = [{'label': "All Utterances", 'value': "all"}] + \
[{'label': ww, 'value': ww} for ww in utts] [{'label': ww, 'value': ww} for ww in utts]
if back_handler: if back_handler:
@ -200,9 +198,7 @@ def utt_opts(back_handler=None, uuid=None):
utt_menu(back_handler=back_handler) utt_menu(back_handler=back_handler)
def _render_ww_tagger(selected_idx, selected_wws, db=None, untagged_only=False): def _render_ww_tagger(selected_idx, selected_wws, untagged_only=False):
db = db or JsonWakeWordDatabase()
def on_tag(tag): def on_tag(tag):
nonlocal selected_idx, selected_wws nonlocal selected_idx, selected_wws
@ -219,21 +215,19 @@ def _render_ww_tagger(selected_idx, selected_wws, db=None, untagged_only=False):
return on_tag(tag) # recurse return on_tag(tag) # recurse
elif selected_idx is not None: elif selected_idx is not None:
db_id = selected_wws[selected_idx]["wakeword_id"] db_id = selected_wws[selected_idx]["recording_id"]
db[db_id]["tag"] = selected_wws[selected_idx]["tag"] = tag DB.update_ww_recording(db_id, tag=tag)
db.commit()
_render_ww_tagger(selected_idx, selected_wws, db, untagged_only=untagged_only) _render_ww_tagger(selected_idx, selected_wws, untagged_only=untagged_only)
def on_gender(tag): def on_gender(tag):
nonlocal selected_idx, selected_wws nonlocal selected_idx, selected_wws
if selected_idx is not None: if selected_idx is not None:
db_id = selected_wws[selected_idx]["wakeword_id"] db_id = selected_wws[selected_idx]["recording_id"]
db[db_id]["speaker_type"] = selected_wws[selected_idx]["speaker_type"] = tag DB.update_ww_recording(db_id, speaker_type=tag)
db.commit()
_render_ww_tagger(selected_idx, selected_wws, db, untagged_only=untagged_only) _render_ww_tagger(selected_idx, selected_wws, untagged_only=untagged_only)
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
content = open(selected_wws[selected_idx]["path"], 'rb').read() content = open(selected_wws[selected_idx]["path"], 'rb').read()
@ -258,8 +252,6 @@ def ww_tagger(back_handler=None, selected_wws=None, selected_idx=None, untagged_
img = open(f'{os.path.dirname(__file__)}/res/wakewords.png', 'rb').read() img = open(f'{os.path.dirname(__file__)}/res/wakewords.png', 'rb').read()
put_image(img) put_image(img)
db = JsonWakeWordDatabase()
def get_next_untagged(): def get_next_untagged():
nonlocal selected_idx nonlocal selected_idx
if untagged_only: if untagged_only:
@ -271,7 +263,7 @@ def ww_tagger(back_handler=None, selected_wws=None, selected_idx=None, untagged_
selected_idx = 0 selected_idx = 0
if not selected_wws: if not selected_wws:
wws = set([w["transcription"] for w in db wws = set([w["transcription"] for w in DB.list_ww_recordings()
if os.path.isfile(w["path"])]) if os.path.isfile(w["path"])])
if not len(wws): if not len(wws):
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
@ -279,7 +271,7 @@ def ww_tagger(back_handler=None, selected_wws=None, selected_idx=None, untagged_
datasets_menu(back_handler=back_handler) datasets_menu(back_handler=back_handler)
return return
current_ww = select("Target WW", wws) current_ww = select("Target WW", wws)
selected_wws = [w for w in db selected_wws = [w for w in DB.list_ww_recordings()
if w["transcription"] == current_ww if w["transcription"] == current_ww
and os.path.isfile(w["path"])] and os.path.isfile(w["path"])]
selected_idx = 0 selected_idx = 0
@ -296,7 +288,7 @@ def ww_tagger(back_handler=None, selected_wws=None, selected_idx=None, untagged_
if "speaker_type" not in ww: if "speaker_type" not in ww:
selected_wws[idx]["speaker_type"] = "untagged" selected_wws[idx]["speaker_type"] = "untagged"
_render_ww_tagger(selected_idx, selected_wws, db, untagged_only) _render_ww_tagger(selected_idx, selected_wws, untagged_only)
buttons = [ buttons = [
{'label': "Show all recordings" if untagged_only else 'Show untagged only', 'value': "toggle"}, {'label': "Show all recordings" if untagged_only else 'Show untagged only', 'value': "toggle"},
@ -326,10 +318,9 @@ def ww_tagger(back_handler=None, selected_wws=None, selected_idx=None, untagged_
for ww in selected_wws: for ww in selected_wws:
if os.path.isfile(ww["path"]): if os.path.isfile(ww["path"]):
os.remove(ww["path"]) os.remove(ww["path"])
dbid = db.get_item_id(ww)
if dbid >= 0: rec_id = current_ww # TODO - rec_id
db.remove_item(dbid) DB.delete_ww_recording(rec_id)
db.commit()
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_text(f"{current_ww} database deleted!") put_text(f"{current_ww} database deleted!")
@ -382,11 +373,9 @@ def ww_menu(back_handler=None):
os.makedirs(f"{CONFIGURATION['data_path']}/wakewords", exist_ok=True) os.makedirs(f"{CONFIGURATION['data_path']}/wakewords", exist_ok=True)
uuid = "AnonDevice" # TODO - allow tagging to a device uuid = "AnonDevice" # TODO - allow tagging to a device
wav_path = f"{CONFIGURATION['data_path']}/wakewords/{name}.{filename}"
meta_path = f"{CONFIGURATION['data_path']}/wakewords/{name}.{filename}.meta"
meta = { meta = {
"transcription": name, "transcription": name,
"path": wav_path,
"meta": { "meta": {
"name": name, "name": name,
"time": time.time(), "time": time.time(),
@ -397,12 +386,8 @@ def ww_menu(back_handler=None):
}, },
"uuid": uuid "uuid": uuid
} }
with JsonWakeWordDatabase() as db: rec = DB.add_ww_recording(byte_data=content, transcription=name, metadata=meta)
db.add_wakeword(name, wav_path, meta, uuid)
with open(wav_path, "wb") as f:
f.write(content)
with open(meta_path, "w") as f:
json.dump(meta, f)
with popup("wake word uploaded!"): with popup("wake word uploaded!"):
put_code(json.dumps(meta, indent=4), "json") put_code(json.dumps(meta, indent=4), "json")
@ -415,15 +400,13 @@ def ww_menu(back_handler=None):
buttons=[{'label': "yes", 'value': True}, buttons=[{'label': "yes", 'value': True},
{'label': "no", 'value': False}]) {'label': "no", 'value': False}])
if opt: if opt:
# remove ww files from path
db = JsonWakeWordDatabase() for rec in DB.list_ww_recordings():
for ww in db: DB.delete_ww_recording(rec_id=rec["recording_id"])
if os.path.isfile(ww["path"]):
os.remove(ww["path"])
# remove db itself
os.remove(db.db.path)
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_text("wake word database deleted!") put_text("wake word database deleted!")
datasets_menu(back_handler=back_handler) datasets_menu(back_handler=back_handler)
return return
if opt == "main": if opt == "main":
@ -469,18 +452,12 @@ def utt_menu(back_handler=None):
os.makedirs(f"{CONFIGURATION['data_path']}/utterances", exist_ok=True) os.makedirs(f"{CONFIGURATION['data_path']}/utterances", exist_ok=True)
uuid = "AnonDevice" # TODO - allow tagging to a device uuid = "AnonDevice" # TODO - allow tagging to a device
path = f"{CONFIGURATION['data_path']}/utterances/{utterance}.{filename}"
meta = { meta = {
"transcription": utterance, "transcription": utterance,
"path": path,
"uuid": uuid "uuid": uuid
} }
with JsonUtteranceDatabase() as db: DB.add_stt_recording(content, utterance, meta)
db.add_utterance(utterance, path, uuid)
with open(path, "wb") as f:
f.write(content)
with popup("utterance recording uploaded!"): with popup("utterance recording uploaded!"):
put_code(json.dumps(meta, indent=4), "json") put_code(json.dumps(meta, indent=4), "json")
@ -494,8 +471,10 @@ def utt_menu(back_handler=None):
buttons=[{'label': "yes", 'value': True}, buttons=[{'label': "yes", 'value': True},
{'label': "no", 'value': False}]) {'label': "no", 'value': False}])
if opt: if opt:
# TODO - also remove files from path
os.remove(JsonUtteranceDatabase().db.path) for rec in DB.list_stt_recordings():
DB.delete_stt_recording(rec["recording_id"])
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_text("utterance database deleted!") put_text("utterance database deleted!")
datasets_menu(back_handler=back_handler) datasets_menu(back_handler=back_handler)

77
ovos_backend_manager/devices.py
View file

@ -3,10 +3,9 @@ import os
import time import time
from uuid import uuid4 from uuid import uuid4
from ovos_local_backend.configuration import CONFIGURATION from ovos_backend_manager.configuration import CONFIGURATION, DB
from ovos_local_backend.database.settings import DeviceDatabase import random
from ovos_local_backend.utils import generate_code from ovos_backend_client.api import GeolocationApi
from ovos_local_backend.utils.geolocate import get_location_config
from pywebio.input import textarea, select, actions, checkbox from pywebio.input import textarea, select, actions, checkbox
from pywebio.output import put_text, put_table, put_markdown, popup, put_code, use_scope, put_image from pywebio.output import put_text, put_table, put_markdown, popup, put_code, use_scope, put_image
@ -52,14 +51,12 @@ def device_menu(uuid, back_handler=None):
['Time Format', d.time_format], ['Time Format', d.time_format],
['System Unit', d.system_unit], ['System Unit', d.system_unit],
['Opt In', d.opt_in], ['Opt In', d.opt_in],
['Selene Blocked', uuid in CONFIGURATION["selene"]["opt_in_blacklist"]],
['Lang', d.lang], ['Lang', d.lang],
['Default Wake Word', d.default_ww], ['Default Wake Word', d.default_ww],
['Default Voice', d.default_tts] ['Default Voice', d.default_tts]
]) ])
db = DeviceDatabase() device = DB.get_device(uuid)
device = db.get_device(uuid)
if device: if device:
y = False y = False
opt = actions(label="What would you like to do?", opt = actions(label="What would you like to do?",
@ -77,74 +74,63 @@ def device_menu(uuid, back_handler=None):
buttons=[{'label': "yes", 'value': True}, buttons=[{'label': "yes", 'value': True},
{'label': "no", 'value': False}]) {'label': "no", 'value': False}])
if y: if y:
db.delete_device(uuid) DB.delete_device(uuid)
db.store()
elif opt == "opt-in": elif opt == "opt-in":
opt_in = checkbox("Open Dataset - device metrics and speech recordings", opt_in = checkbox("Open Dataset - device metrics and speech recordings",
[{'label': 'Store metrics and recordings', [{'label': 'Store metrics and recordings',
'selected': device.opt_in, 'selected': device.opt_in,
'value': "opt_in"}, 'value': "opt_in"}])
{'label': 'Block Selene sharing',
'selected': uuid in CONFIGURATION["selene"]["opt_in_blacklist"], device["opt_in"] = "opt_in" in opt_in
'value': "blacklist"}])
device.opt_in = "opt_in" in opt_in
if "blacklist" in opt_in:
if uuid not in CONFIGURATION["selene"]["opt_in_blacklist"]:
CONFIGURATION["selene"]["opt_in_blacklist"].append(uuid)
CONFIGURATION.store()
else:
if uuid in CONFIGURATION["selene"]["opt_in_blacklist"]:
CONFIGURATION["selene"]["opt_in_blacklist"].remove(uuid)
CONFIGURATION.store()
elif opt == "tts": elif opt == "tts":
tts = select("Choose a voice", tts = select("Choose a voice",
list(CONFIGURATION["tts_configs"].keys())) list(CONFIGURATION["tts_configs"].keys()))
device.default_tts = CONFIGURATION["tts_configs"][tts]["module"] device["default_tts"] = CONFIGURATION["tts_configs"][tts]["module"]
device.default_tts_cfg = CONFIGURATION["tts_configs"][tts] device["default_tts_cfg"] = CONFIGURATION["tts_configs"][tts]
elif opt == "ww": elif opt == "ww":
ww = select("Choose a wake word", ww = select("Choose a wake word",
list(CONFIGURATION["ww_configs"].keys())) list(CONFIGURATION["ww_configs"].keys()))
device.default_ww = ww device["default_ww"] = ww
device.default_ww_cfg = CONFIGURATION["ww_configs"][ww] device["default_ww_cfg"] = CONFIGURATION["ww_configs"][ww]
elif opt == "date": elif opt == "date":
date = select("Change date format", date = select("Change date format",
['DMY', 'MDY']) ['DMY', 'MDY'])
device.date_format = date device["date_format"] = date
elif opt == "time": elif opt == "time":
tim = select("Change time format", tim = select("Change time format",
['full', 'short']) ['full', 'short'])
device.time_format = tim device["time_format"] = tim
elif opt == "unit": elif opt == "unit":
unit = select("Change system units", unit = select("Change system units",
['metric', 'imperial']) ['metric', 'imperial'])
device.system_unit = unit device["system_unit"] = unit
elif opt == "email": elif opt == "email":
email = textarea("Enter your device email", email = textarea("Enter your device email",
placeholder="notify@me.com", placeholder="notify@me.com",
required=True) required=True)
device.email = email device["email"] = email
elif opt == "name": elif opt == "name":
name = textarea("Enter your device name", name = textarea("Enter your device name",
placeholder="OVOS Mark2", placeholder="OVOS Mark2",
required=True) required=True)
device.name = name device["name"] = name
elif opt == "location": elif opt == "location":
loc = textarea("Enter your device placement", loc = textarea("Enter your device placement",
placeholder="kitchen", placeholder="kitchen",
required=True) required=True)
device.device_location = loc device["device_location"] = loc
elif opt == "geo": elif opt == "geo":
loc = textarea("Enter an address", loc = textarea("Enter an address",
placeholder="Anywhere street Any city Nº234", placeholder="Anywhere street Any city Nº234",
required=True) required=True)
data = get_location_config(loc) data = GeolocationApi().get_geolocation(loc)
device.location = data device["location"] = data
elif opt == "identity": elif opt == "identity":
identity = {"uuid": device.uuid, identity = {"uuid": device["uuid"],
"expires_at": time.time() + 99999999999999, "expires_at": time.time() + 99999999999999,
"accessToken": device.token, "accessToken": device["token"],
"refreshToken": device.token} "refreshToken": device["token"]}
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_markdown(f'### identity2.json') put_markdown(f'### identity2.json')
put_code(json.dumps(identity, indent=4), "json") put_code(json.dumps(identity, indent=4), "json")
@ -154,8 +140,7 @@ def device_menu(uuid, back_handler=None):
update_info(device, False) update_info(device, False)
if opt not in ["identity", "delete", "view_loc"]: if opt not in ["identity", "delete", "view_loc"]:
db.update_device(device) DB.update_device(**device)
db.store()
popup("Device updated!") popup("Device updated!")
elif opt == "delete" and y: elif opt == "delete" and y:
uuid = None uuid = None
@ -172,8 +157,8 @@ def device_select(back_handler=None):
img = open(f'{os.path.dirname(__file__)}/res/devices.png', 'rb').read() img = open(f'{os.path.dirname(__file__)}/res/devices.png', 'rb').read()
put_image(img) put_image(img)
devices = {uuid: f"{device['name']}@{device['device_location']}" devices = {device["uuid"]: f"{device['name']}@{device['device_location']}"
for uuid, device in DeviceDatabase().items()} for device in DB.list_devices()}
buttons = [{'label': d, 'value': uuid} for uuid, d in devices.items()] + \ buttons = [{'label': d, 'value': uuid} for uuid, d in devices.items()] + \
[{'label': 'Delete device database', 'value': "delete_devices"}] [{'label': 'Delete device database', 'value': "delete_devices"}]
if back_handler: if back_handler:
@ -195,7 +180,9 @@ def device_select(back_handler=None):
buttons=[{'label': "yes", 'value': True}, buttons=[{'label': "yes", 'value': True},
{'label': "no", 'value': False}]) {'label': "no", 'value': False}])
if opt: if opt:
os.remove(DeviceDatabase().path) for dev in DB.list_devices():
DB.delete_device(dev["uuid"])
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
if back_handler: if back_handler:
back_handler() back_handler()
@ -218,11 +205,11 @@ def instant_pair(back_handler=None):
put_image(img) put_image(img)
uuid = str(uuid4()) uuid = str(uuid4())
code = generate_code() code = f"{random.randint(100, 999)}ABC"
token = f"{code}:{uuid}" token = f"{code}:{uuid}"
# add device to db # add device to db
with DeviceDatabase() as db: DB.add_device(uuid, token)
db.add_device(uuid, token)
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_markdown("# Device paired!") put_markdown("# Device paired!")

8
ovos_backend_manager/menu.py
View file

@ -1,4 +1,4 @@
from ovos_local_backend.configuration import CONFIGURATION from ovos_backend_manager.configuration import CONFIGURATION
from pywebio.input import textarea, actions from pywebio.input import textarea, actions
from pywebio.output import put_text, popup, use_scope, put_image from pywebio.output import put_text, popup, use_scope, put_image
@ -8,7 +8,6 @@ from ovos_backend_manager.devices import device_select, instant_pair
from ovos_backend_manager.metrics import metrics_menu from ovos_backend_manager.metrics import metrics_menu
from ovos_backend_manager.microservices import microservices_menu from ovos_backend_manager.microservices import microservices_menu
from ovos_backend_manager.oauth import oauth_menu from ovos_backend_manager.oauth import oauth_menu
from ovos_backend_manager.selene import selene_menu
def main_menu(): def main_menu():
@ -24,8 +23,7 @@ def main_menu():
{'label': 'Manage Datasets', 'value': "db"}, {'label': 'Manage Datasets', 'value': "db"},
{'label': 'OAuth Applications', 'value': "oauth"}, {'label': 'OAuth Applications', 'value': "oauth"},
{'label': 'Configure Backend', 'value': "backend"}, {'label': 'Configure Backend', 'value': "backend"},
{'label': 'Configure Microservices', 'value': "services"}, {'label': 'Configure Microservices', 'value': "services"}])
{'label': 'Configure Selene Proxy', 'value': "selene"}])
if opt == "pair": if opt == "pair":
instant_pair(back_handler=main_menu) instant_pair(back_handler=main_menu)
elif opt == "services": elif opt == "services":
@ -36,8 +34,6 @@ def main_menu():
datasets_menu(back_handler=main_menu) datasets_menu(back_handler=main_menu)
elif opt == "backend": elif opt == "backend":
backend_menu(back_handler=main_menu) backend_menu(back_handler=main_menu)
elif opt == "selene":
selene_menu(back_handler=main_menu)
elif opt == "device": elif opt == "device":
device_select(back_handler=main_menu) device_select(back_handler=main_menu)
elif opt == "metrics": elif opt == "metrics":

62
ovos_backend_manager/metrics.py
View file

@ -3,10 +3,8 @@ import os
import time import time
from cutecharts.charts import Pie, Bar, Scatter from cutecharts.charts import Pie, Bar, Scatter
from ovos_local_backend.database.metrics import JsonMetricDatabase
from ovos_local_backend.database.settings import DeviceDatabase from ovos_backend_manager.configuration import CONFIGURATION, DB
from ovos_local_backend.database.utterances import JsonUtteranceDatabase
from ovos_local_backend.database.wakewords import JsonWakeWordDatabase
from pywebio.input import actions from pywebio.input import actions
from pywebio.output import put_text, popup, put_code, put_markdown, put_html, use_scope, put_image from pywebio.output import put_text, popup, put_code, put_markdown, put_html, use_scope, put_image
@ -14,8 +12,8 @@ chart_type = Pie
def device_select(back_handler=None): def device_select(back_handler=None):
devices = {uuid: f"{device['name']}@{device['device_location']}" devices = {device["uuid"]: f"{device['name']}@{device['device_location']}"
for uuid, device in DeviceDatabase().items()} for device in DB.list_devices()}
buttons = [{'label': "All Devices", 'value': "all"}] + \ buttons = [{'label': "All Devices", 'value': "all"}] + \
[{'label': d, 'value': uuid} for uuid, d in devices.items()] [{'label': d, 'value': uuid} for uuid, d in devices.items()]
if back_handler: if back_handler:
@ -41,29 +39,29 @@ def device_select(back_handler=None):
def metrics_select(back_handler=None, uuid=None): def metrics_select(back_handler=None, uuid=None):
buttons = [] buttons = []
db = JsonMetricDatabase() metrics = DB.list_metrics()
if not len(db): if not len(metrics):
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_text("No metrics uploaded yet!") put_text("No metrics uploaded yet!")
metrics_menu(back_handler=back_handler, uuid=uuid) metrics_menu(back_handler=back_handler, uuid=uuid)
return return
for m in db: for m in metrics:
name = f"{m['metric_id']}-{m['metric_type']}" name = f"{m['metric_id']}-{m['metric_type']}"
if uuid is not None and m["uuid"] != uuid: if uuid is not None and m["uuid"] != uuid:
continue continue
buttons.append({'label': name, 'value': m['metric_id']}) buttons.append({'label': name, 'value': m['metric_id']})
if back_handler: if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
opt = actions(label="Select a metric to inspect", metric_id = actions(label="Select a metric to inspect",
buttons=buttons) buttons=buttons)
if opt == "main": if metric_id == "main":
device_select(back_handler=back_handler) device_select(back_handler=back_handler)
return return
# id == db_position + 1
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_markdown("# Metadata") put_markdown("# Metadata")
put_code(json.dumps(db[opt - 1], indent=4), "json") put_code(json.dumps(metric_id, indent=4), "json")
metrics_select(back_handler=back_handler, uuid=uuid) metrics_select(back_handler=back_handler, uuid=uuid)
@ -167,8 +165,8 @@ def _plot_metrics(uuid, selected_metric="types"):
md = "" md = ""
if uuid is None: if uuid is None:
md = f"""# Open Dataset Report md = f"""# Open Dataset Report
Total Registered Devices: {len(DeviceDatabase())} Total Registered Devices: {len(DB.list_devices())}
Currently Opted-in: {len([d for d in DeviceDatabase() if d.opt_in])} Currently Opted-in: {len([d for d in DB.list_devices() if d["opt_in"]])}
Unique Devices seen: {m.total_devices}""" Unique Devices seen: {m.total_devices}"""
# Open Dataset Report""" # Open Dataset Report"""
@ -239,7 +237,9 @@ def metrics_menu(back_handler=None, uuid=None, selected_metric="types"):
buttons=[{'label': "yes", 'value': True}, buttons=[{'label': "yes", 'value': True},
{'label': "no", 'value': False}]) {'label': "no", 'value': False}])
if opt: if opt:
os.remove(JsonMetricDatabase().db.path) for m in DB.list_metrics():
DB.delete_metric(m["metric_id"])
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
if back_handler: if back_handler:
back_handler() back_handler()
@ -264,10 +264,10 @@ class MetricsReportGenerator:
self.total_fallbacks = 0 self.total_fallbacks = 0
self.total_stt = 0 self.total_stt = 0
self.total_tts = 0 self.total_tts = 0
self.total_ww = len(JsonWakeWordDatabase()) self.total_ww = len(DB.list_ww_recordings())
self.total_utt = len(JsonUtteranceDatabase()) self.total_utt = len(DB.list_stt_recordings())
self.total_devices = len(DeviceDatabase()) self.total_devices = len(DB.list_devices())
self.total_metrics = len(JsonMetricDatabase()) self.total_metrics = len(DB.list_metrics())
self.intents = {} self.intents = {}
self.fallbacks = {} self.fallbacks = {}
@ -289,10 +289,10 @@ class MetricsReportGenerator:
self.total_fallbacks = 0 self.total_fallbacks = 0
self.total_stt = 0 self.total_stt = 0
self.total_tts = 0 self.total_tts = 0
self.total_ww = len(JsonWakeWordDatabase()) self.total_ww = len(DB.list_ww_recordings())
self.total_metrics = len(JsonMetricDatabase()) self.total_utt = len(DB.list_stt_recordings())
self.total_utt = len(JsonUtteranceDatabase())
self.total_devices = 0 self.total_devices = 0
self.total_metrics = len(DB.list_metrics())
self.intents = {} self.intents = {}
self.devices = {} self.devices = {}
@ -309,11 +309,11 @@ class MetricsReportGenerator:
def load_metrics(self): def load_metrics(self):
self.reset_metrics() self.reset_metrics()
for m in JsonMetricDatabase(): for m in DB.list_metrics():
if m["uuid"] not in self.devices: if m["uuid"] not in self.devices:
self.total_devices += 1 self.total_devices += 1
self._process_metric(m) self._process_metric(m)
for ww in JsonWakeWordDatabase(): for ww in DB.list_ww_recordings():
if ww["meta"]["name"] not in self.ww: if ww["meta"]["name"] not in self.ww:
self.ww[ww["meta"]["name"]] = 0 self.ww[ww["meta"]["name"]] = 0
else: else:
@ -333,7 +333,7 @@ class MetricsReportGenerator:
@property @property
def untracked_devices(self): def untracked_devices(self):
return [dev.uuid for dev in DeviceDatabase() if not dev.opt_in] return [dev["uuid"] for dev in DB.list_devices() if not dev["opt_in"]]
# cute charts # cute charts
def timings_chart(self): def timings_chart(self):
@ -577,18 +577,18 @@ class DeviceMetricsReportGenerator(MetricsReportGenerator):
def load_metrics(self): def load_metrics(self):
self.reset_metrics() self.reset_metrics()
self.total_ww = len([ww for ww in JsonWakeWordDatabase() self.total_ww = len([ww for ww in DB.list_ww_recordings()
if ww["uuid"] == self.uuid]) if ww["uuid"] == self.uuid])
self.total_metrics = 0 self.total_metrics = 0
self.total_utt = len([utt for utt in JsonUtteranceDatabase() self.total_utt = len([utt for utt in DB.list_stt_recordings()
if utt["uuid"] == self.uuid]) if utt["uuid"] == self.uuid])
for m in JsonMetricDatabase(): for m in DB.list_metrics():
if m["uuid"] != self.uuid: if m["uuid"] != self.uuid:
continue continue
self._process_metric(m) self._process_metric(m)
self.total_metrics += 1 self.total_metrics += 1
for ww in JsonWakeWordDatabase(): for ww in DB.list_ww_recordings():
if ww["uuid"] != self.uuid: if ww["uuid"] != self.uuid:
continue continue
if ww["meta"]["name"] not in self.ww: if ww["meta"]["name"] not in self.ww:
@ -598,5 +598,5 @@ class DeviceMetricsReportGenerator(MetricsReportGenerator):
if __name__ == "__main__": if __name__ == "__main__":
for ww in JsonWakeWordDatabase(): for ww in DB.list_ww_recordings():
print(ww) print(ww)

24
ovos_backend_manager/microservices.py
View file

@ -1,7 +1,7 @@
import json import json
import os import os
from ovos_local_backend.configuration import CONFIGURATION from ovos_backend_manager.configuration import CONFIGURATION
from ovos_plugin_manager.stt import get_stt_configs, get_stt_supported_langs, get_stt_lang_configs from ovos_plugin_manager.stt import get_stt_configs, get_stt_supported_langs, get_stt_lang_configs
from pywebio.input import select, actions, input_group, input, TEXT, NUMBER from pywebio.input import select, actions, input_group, input, TEXT, NUMBER
from pywebio.output import put_text, put_table, popup, put_code, put_image, use_scope from pywebio.output import put_text, put_table, popup, put_code, put_image, use_scope
@ -11,8 +11,7 @@ def _get_stt_opts(lang=None):
STT_CONFIGS = {} STT_CONFIGS = {}
if lang is not None: if lang is not None:
for p, data in get_stt_lang_configs(lang, include_dialects=True).items(): for p, data in get_stt_lang_configs(lang, include_dialects=True).items():
if p == "ovos-stt-plugin-selene" and \ if p == "ovos-stt-plugin-selene":
not CONFIGURATION["selene"].get("enabled"):
continue continue
if not data: if not data:
continue continue
@ -22,8 +21,7 @@ def _get_stt_opts(lang=None):
STT_CONFIGS[cfg["display_name"]] = cfg STT_CONFIGS[cfg["display_name"]] = cfg
else: else:
for p, data in get_stt_configs().items(): for p, data in get_stt_configs().items():
if p == "ovos-stt-plugin-selene" and \ if p == "ovos-stt-plugin-selene":
not CONFIGURATION["selene"].get("enabled"):
continue continue
if not data: if not data:
continue continue
@ -40,8 +38,6 @@ def microservices_menu(back_handler=None):
img = open(f'{os.path.dirname(__file__)}/res/microservices_config.png', 'rb').read() img = open(f'{os.path.dirname(__file__)}/res/microservices_config.png', 'rb').read()
put_image(img) put_image(img)
selene = CONFIGURATION["selene"]["enabled"]
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
put_table([ put_table([
['STT module', CONFIGURATION["stt"]["module"]], ['STT module', CONFIGURATION["stt"]["module"]],
@ -49,12 +45,7 @@ def microservices_menu(back_handler=None):
['WolframAlpha provider', CONFIGURATION["microservices"]["wolfram_provider"]], ['WolframAlpha provider', CONFIGURATION["microservices"]["wolfram_provider"]],
['Weather provider', CONFIGURATION["microservices"]["weather_provider"]], ['Weather provider', CONFIGURATION["microservices"]["weather_provider"]],
['Geolocation provider', CONFIGURATION["microservices"]["geolocation_provider"]], ['Geolocation provider', CONFIGURATION["microservices"]["geolocation_provider"]]
['Selene WolframAlpha proxy enabled', selene and CONFIGURATION["selene"]["proxy_wolfram"]],
['Selene OpenWeatherMap proxy enabled', selene and CONFIGURATION["selene"]["proxy_weather"]],
['Selene Geolocation proxy enabled', selene and CONFIGURATION["selene"]["proxy_geolocation"]],
['Selene Email proxy enabled', selene and CONFIGURATION["selene"]["proxy_email"]]
]) ])
buttons = [{'label': 'Configure STT', 'value': "stt"}, buttons = [{'label': 'Configure STT', 'value': "stt"},
@ -79,8 +70,6 @@ def microservices_menu(back_handler=None):
return return
elif opt == "geo": elif opt == "geo":
opts = ["OpenStreetMap", "ArcGIS", "Geocode Farm"] # TODO - ovos endpoint opts = ["OpenStreetMap", "ArcGIS", "Geocode Farm"] # TODO - ovos endpoint
if selene and CONFIGURATION["selene"]["proxy_geolocation"]:
opts.append("Selene")
provider = select("Choose a weather provider", opts) provider = select("Choose a weather provider", opts)
if provider == "OpenStreetMap": if provider == "OpenStreetMap":
provider = "osm" provider = "osm"
@ -89,16 +78,12 @@ def microservices_menu(back_handler=None):
opts = ["ovos"] opts = ["ovos"]
if CONFIGURATION["microservices"]["owm_key"]: if CONFIGURATION["microservices"]["owm_key"]:
opts.append("local") opts.append("local")
if selene and CONFIGURATION["selene"]["proxy_weather"]:
opts.append("selene")
provider = select("Choose a weather provider", opts) provider = select("Choose a weather provider", opts)
CONFIGURATION["microservices"]["weather_provider"] = provider CONFIGURATION["microservices"]["weather_provider"] = provider
elif opt == "wolfram": elif opt == "wolfram":
opts = ["ovos"] opts = ["ovos"]
if CONFIGURATION["microservices"]["wolfram_key"]: if CONFIGURATION["microservices"]["wolfram_key"]:
opts.append("local") opts.append("local")
if selene and CONFIGURATION["selene"]["proxy_wolfram"]:
opts.append("selene")
provider = select("Choose a WolframAlpha provider", opts) provider = select("Choose a WolframAlpha provider", opts)
CONFIGURATION["microservices"]["wolfram_provider"] = provider CONFIGURATION["microservices"]["wolfram_provider"] = provider
elif opt == "ovos": elif opt == "ovos":
@ -133,7 +118,6 @@ def microservices_menu(back_handler=None):
CONFIGURATION["microservices"]["owm_key"] = data["owm"] CONFIGURATION["microservices"]["owm_key"] = data["owm"]
popup("Secrets updated!") popup("Secrets updated!")
elif opt == "smtp": elif opt == "smtp":
# TODO - checkbox for selene proxy
# TODO - ovos endpoint # TODO - ovos endpoint
if "smtp" not in CONFIGURATION["email"]: if "smtp" not in CONFIGURATION["email"]:

39
ovos_backend_manager/oauth.py
View file

@ -2,8 +2,7 @@ import json
import os import os
from oauthlib.oauth2 import WebApplicationClient from oauthlib.oauth2 import WebApplicationClient
from ovos_local_backend.configuration import CONFIGURATION from ovos_backend_manager.configuration import CONFIGURATION, DB
from ovos_local_backend.database.oauth import OAuthApplicationDatabase, OAuthTokenDatabase
from pywebio.input import actions, input_group, input, TEXT from pywebio.input import actions, input_group, input, TEXT
from pywebio.output import use_scope, popup, put_image, put_link, put_code, put_text, put_table, put_markdown from pywebio.output import use_scope, popup, put_image, put_link, put_code, put_text, put_table, put_markdown
@ -11,7 +10,7 @@ from pywebio.output import use_scope, popup, put_image, put_link, put_code, put_
def get_oauth_data(app_id=None): def get_oauth_data(app_id=None):
data = {} data = {}
if app_id: if app_id:
data = OAuthApplicationDatabase().get(app_id) data = DB.get_oauth_app(app_id)
data = data or {'auth_endpoint': "https://", data = data or {'auth_endpoint': "https://",
'token_endpoint': "https://", 'token_endpoint': "https://",
'refresh_endpoint': "https://"} 'refresh_endpoint': "https://"}
@ -56,7 +55,7 @@ def authorize_app(data):
def _render_app(app_id): def _render_app(app_id):
with use_scope("main_view", clear=True): with use_scope("main_view", clear=True):
data = OAuthApplicationDatabase()[app_id] data = DB.get_oauth_app(app_id)
put_markdown(f'# {app_id.title()}') put_markdown(f'# {app_id.title()}')
put_table([ put_table([
@ -83,7 +82,7 @@ def app_menu(app_id, back_handler=None):
buttons = [ buttons = [
{'label': "Configure", 'value': "oauth"}, {'label': "Configure", 'value': "oauth"},
] ]
tok = OAuthTokenDatabase().get(app_id) tok = DB.get_oauth_token(app_id)
if tok: if tok:
buttons.append({'label': "View Token", 'value': "token"}) buttons.append({'label': "View Token", 'value': "token"})
buttons.append({'label': "Refresh Token", 'value': "refresh"}) buttons.append({'label': "Refresh Token", 'value': "refresh"})
@ -103,22 +102,17 @@ def app_menu(app_id, back_handler=None):
with popup("OAuth Token"): with popup("OAuth Token"):
put_code(json.dumps(tok, indent=4), language="json") put_code(json.dumps(tok, indent=4), language="json")
elif opt == "auth" or opt == "refresh": # TODO special refresh handling (?) elif opt == "auth" or opt == "refresh": # TODO special refresh handling (?)
data = OAuthApplicationDatabase()[app_id] data = DB.get_oauth_app(app_id)
authorize_app(data) authorize_app(data)
elif opt == "oauth": elif opt == "oauth":
data = get_oauth_data(app_id) data = get_oauth_data(app_id)
app_id = data.pop("oauth_service")
with OAuthApplicationDatabase() as db: DB.update_oauth_app(app_id, **data)
db[data["oauth_service"]] = data
with popup(app_id): with popup(app_id):
put_text(f"{app_id} oauth settings updated!") put_text(f"{app_id} oauth settings updated!")
elif opt == "delete": elif opt == "delete":
db = OAuthApplicationDatabase() DB.delete_oauth_app(app_id)
if app_id in db:
db.pop(app_id)
db.store()
with popup(app_id): with popup(app_id):
put_text(f"{app_id} oauth settings deleted!") put_text(f"{app_id} oauth settings deleted!")
oauth_menu(back_handler=back_handler) oauth_menu(back_handler=back_handler)
@ -138,8 +132,10 @@ def oauth_menu(back_handler=None):
pass pass
buttons = [{'label': 'New App', 'value': "new"}] buttons = [{'label': 'New App', 'value': "new"}]
for app, data in OAuthApplicationDatabase().items(): for app in DB.list_oauth_apps():
buttons.append({'label': app, 'value': app}) app_id = app.pop("token_id")
data = app
buttons.append({'label': app_id, 'value': data})
if back_handler: if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
@ -147,15 +143,8 @@ def oauth_menu(back_handler=None):
buttons=buttons) buttons=buttons)
if opt == "new": if opt == "new":
data = get_oauth_data() data = get_oauth_data()
app_id = data.pop("oauth_service")
with OAuthApplicationDatabase() as db: DB.add_oauth_app(app_id, **data)
db.add_application(data["oauth_service"],
data["client_id"],
data["client_secret"],
data["auth_endpoint"],
data["token_endpoint"],
data["refresh_endpoint"],
data["scope"])
authorize_app(data) authorize_app(data)

303
ovos_backend_manager/selene.py
View file

@ -1,303 +0,0 @@
import json
import os
from ovos_local_backend.configuration import CONFIGURATION
from pywebio.input import actions, file_upload, input_group, textarea
from pywebio.output import put_table, popup, use_scope, put_image, put_markdown, put_code
def pairing_menu(back_handler=None):
version = CONFIGURATION["selene"]["version"]
host = CONFIGURATION["selene"]["url"]
ident = CONFIGURATION["selene"]["identity_file"]
paired = os.path.exists(ident)
with use_scope("main_view", clear=True):
put_markdown("# Status")
put_table([
['Enabled', CONFIGURATION["selene"]["enabled"]],
['Host', host],
['Version', version],
['Identity', ident],
['Paired', paired],
['Proxy Pairing Enabled', CONFIGURATION["selene"]["proxy_pairing"]]
])
if os.path.isfile(ident):
with open(ident) as f:
content = f.read()
put_markdown("# Identity")
put_code(content, "json")
buttons = [{'label': 'Upload identity2.json', 'value': "upload"},
{'label': 'Paste identity2.json', 'value': "paste"}]
if os.path.isfile(ident):
buttons.append({'label': 'Delete identity2.json', 'value': "delete"})
label = "Enable Proxy Pairing" if not CONFIGURATION["selene"]["proxy_pairing"] else "Disable Proxy Pairing"
buttons.append({'label': label, 'value': "proxy"})
if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
opt = actions(label="What would you like to do?", buttons=buttons)
if opt == "main":
selene_menu(back_handler=back_handler)
return
elif opt == "delete":
with use_scope("main_view", clear=True):
if os.path.isfile(ident):
os.remove(ident)
popup("Identity deleted!")
else:
popup("Identity does not exist!")
elif opt == "upload":
with use_scope("main_view", clear=True):
data = input_group("Upload identity", [
file_upload("identity file", name="file")
])
mime = data["file"]["mime_type"]
content = data["file"]["content"]
if mime != "application/json":
popup("invalid format!")
else:
os.makedirs(os.path.dirname(ident), exist_ok=True)
with open(ident, "wb") as f:
f.write(content)
with popup("Identity uploaded!"):
put_code(content.decode("utf-8"), "json")
elif opt == "paste":
with use_scope("main_view", clear=True):
dummy = """{
"uuid": "31628fa1-dbfd-4626-aaa2-1464dd204715",
"expires_at": 100001663862051.53,
"accessToken": "8YI3NQ:31628fa1-dbfd-4626-aaa2-1464dd204715",
"refreshToken": "8YI3NQ:31628fa1-dbfd-4626-aaa2-1464dd204715"
}
"""
data = textarea("identity2.json", placeholder=dummy, required=True)
with open(ident, "w") as f:
f.write(data)
with popup("Identity updated!"):
put_code(data, "json")
elif opt == "proxy":
CONFIGURATION["selene"]["proxy_pairing"] = not CONFIGURATION["selene"]["proxy_pairing"]
CONFIGURATION.store()
pairing_menu(back_handler=back_handler)
def account_menu(back_handler=None):
version = CONFIGURATION["selene"]["version"]
host = CONFIGURATION["selene"]["url"]
ident = CONFIGURATION["selene"]["identity_file"]
paired = os.path.exists(ident)
uuid = None
if paired:
with open(ident) as f:
uuid = json.load(f)["uuid"]
with use_scope("main_view", clear=True):
put_markdown("# Account")
put_table([
['Selene UUID', uuid],
['Download Location', CONFIGURATION["selene"]["download_location"]],
['Download Preferences', CONFIGURATION["selene"]["download_prefs"]],
['Download Skill Settings', CONFIGURATION["selene"]["download_settings"]],
['Upload Skill Settings', CONFIGURATION["selene"]["upload_settings"]],
['Force 2 way Skill Settings sync', CONFIGURATION["selene"]["force2way"]]
])
buttons = []
label = "Enable Location Download" if not CONFIGURATION["selene"][
"download_location"] else "Disable Location Download"
buttons.append({'label': label, 'value': "location"})
label = "Enable Preferences Download" if not CONFIGURATION["selene"][
"download_prefs"] else "Disable Preferences Download"
buttons.append({'label': label, 'value': "prefs"})
label = "Enable SkillSettings Download" if not CONFIGURATION["selene"][
"download_settings"] else "Disable SkillSettings Download"
buttons.append({'label': label, 'value': "download_settings"})
label = "Enable SkillSettings Upload" if not CONFIGURATION["selene"][
"upload_settings"] else "Disable SkillSettings Upload"
buttons.append({'label': label, 'value': "upload_settings"})
label = "Enable forced 2way sync" if not CONFIGURATION["selene"]["force2way"] else "Disable forced 2way sync"
buttons.append({'label': label, 'value': "2way"})
if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
opt = actions(label="What would you like to do?", buttons=buttons)
if opt == "main":
selene_menu(back_handler=back_handler)
return
elif opt == "location":
CONFIGURATION["selene"]["download_location"] = not CONFIGURATION["selene"]["download_location"]
elif opt == "prefs":
CONFIGURATION["selene"]["download_prefs"] = not CONFIGURATION["selene"]["download_prefs"]
elif opt == "download_settings":
CONFIGURATION["selene"]["download_settings"] = not CONFIGURATION["selene"]["download_settings"]
elif opt == "upload_settings":
CONFIGURATION["selene"]["upload_settings"] = not CONFIGURATION["selene"]["upload_settings"]
elif opt == "2way":
CONFIGURATION["selene"]["force2way"] = not CONFIGURATION["selene"]["force2way"]
CONFIGURATION.store()
account_menu(back_handler=back_handler)
def integrations_menu(back_handler=None):
version = CONFIGURATION["selene"]["version"]
host = CONFIGURATION["selene"]["url"]
ident = CONFIGURATION["selene"]["identity_file"]
paired = os.path.exists(ident)
uuid = None
if paired:
with open(ident) as f:
uuid = json.load(f)["uuid"]
with use_scope("main_view", clear=True):
put_markdown("# Integrations")
put_table([
['Selene UUID', uuid],
['Weather Enabled', CONFIGURATION["selene"]["proxy_weather"]],
['WolframAlpha Enabled', CONFIGURATION["selene"]["proxy_wolfram"]],
['Geolocation Enabled', CONFIGURATION["selene"]["proxy_geolocation"]],
['Email Enabled', CONFIGURATION["selene"]["proxy_email"]]
])
buttons = []
label = "Enable Weather Proxy" if not CONFIGURATION["selene"]["proxy_weather"] else "Disable Weather Proxy"
buttons.append({'label': label, 'value': "weather"})
label = "Enable WolframAlpha Proxy" if not CONFIGURATION["selene"][
"proxy_wolfram"] else "Disable WolframAlpha Proxy"
buttons.append({'label': label, 'value': "wolfram"})
label = "Enable Geolocation Proxy" if not CONFIGURATION["selene"][
"proxy_geolocation"] else "Disable Geolocation Proxy"
buttons.append({'label': label, 'value': "geolocation"})
label = "Enable Email Proxy" if not CONFIGURATION["selene"]["proxy_email"] else "Disable Email Proxy"
buttons.append({'label': label, 'value': "email"})
if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
opt = actions(label="What would you like to do?", buttons=buttons)
if opt == "main":
selene_menu(back_handler=back_handler)
return
elif opt == "geolocation":
CONFIGURATION["selene"]["proxy_geolocation"] = not CONFIGURATION["selene"]["proxy_geolocation"]
elif opt == "weather":
CONFIGURATION["selene"]["proxy_weather"] = not CONFIGURATION["selene"]["proxy_weather"]
elif opt == "wolfram":
CONFIGURATION["selene"]["proxy_wolfram"] = not CONFIGURATION["selene"]["proxy_wolfram"]
elif opt == "email":
CONFIGURATION["selene"]["proxy_email"] = not CONFIGURATION["selene"]["proxy_email"]
CONFIGURATION.store()
integrations_menu(back_handler=back_handler)
def dataset_menu(back_handler=None):
version = CONFIGURATION["selene"]["version"]
host = CONFIGURATION["selene"]["url"]
ident = CONFIGURATION["selene"]["identity_file"]
paired = os.path.exists(ident)
with use_scope("main_view", clear=True):
put_markdown("# Open Dataset")
put_table([
['Opt In', CONFIGURATION["selene"]["opt_in"]],
['Upload Metrics', CONFIGURATION["selene"]["upload_metrics"]],
['Upload Wake Words', CONFIGURATION["selene"]["upload_wakewords"]],
['Upload Utterances', CONFIGURATION["selene"]["upload_utterances"]]
])
buttons = []
label = "Enable Open Dataset Opt In" if not CONFIGURATION["selene"]["opt_in"] else "Disable Open Dataset Opt In"
buttons.append({'label': label, 'value': "opt_in"})
label = "Enable Metrics Upload" if not CONFIGURATION["selene"]["upload_metrics"] else "Disable Metrics Upload"
buttons.append({'label': label, 'value': "metrics"})
label = "Enable Wake Words Upload" if not CONFIGURATION["selene"][
"upload_wakewords"] else "Disable Wake Words Upload"
buttons.append({'label': label, 'value': "ww"})
label = "Enable Utterances Upload" if not CONFIGURATION["selene"][
"upload_utterances"] else "Disable Utterances Upload"
buttons.append({'label': label, 'value': "stt"})
if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
opt = actions(label="What would you like to do?", buttons=buttons)
if opt == "main":
selene_menu(back_handler=back_handler)
return
elif opt == "opt_in":
CONFIGURATION["selene"]["opt_in"] = not CONFIGURATION["selene"]["opt_in"]
elif opt == "selene":
CONFIGURATION["selene"]["enabled"] = not CONFIGURATION["selene"]["enabled"]
elif opt == "stt":
CONFIGURATION["selene"]["upload_utterances"] = not CONFIGURATION["selene"]["upload_utterances"]
elif opt == "ww":
CONFIGURATION["selene"]["upload_wakewords"] = not CONFIGURATION["selene"]["upload_wakewords"]
elif opt == "metrics":
CONFIGURATION["selene"]["upload_metrics"] = not CONFIGURATION["selene"]["upload_metrics"]
CONFIGURATION.store()
dataset_menu(back_handler=back_handler)
def selene_menu(back_handler=None):
version = CONFIGURATION["selene"]["version"]
host = CONFIGURATION["selene"]["url"]
ident = CONFIGURATION["selene"]["identity_file"]
paired = os.path.exists(ident)
with use_scope("logo", clear=True):
img = open(f'{os.path.dirname(__file__)}/res/selene_proxy.png', 'rb').read()
put_image(img)
with use_scope("main_view", clear=True):
put_markdown("# Status")
put_table([
['Enabled', CONFIGURATION["selene"]["enabled"]],
['Host', host]
])
buttons = [{'label': 'Manage Identity', 'value': "pair"},
{'label': 'Manage Account', 'value': "account"},
{'label': 'Manage Integrations', 'value': "integrations"},
{'label': 'Manage Open Dataset', 'value': "dataset"}]
if CONFIGURATION["selene"]["enabled"]:
buttons.insert(0, {'label': "Disable Selene", 'value': "selene"})
else:
buttons.insert(0, {'label': "Enable Selene", 'value': "selene"})
if back_handler:
buttons.insert(0, {'label': '<- Go Back', 'value': "main"})
opt = actions(label="What would you like to do?", buttons=buttons)
if opt == "main":
with use_scope("main_view", clear=True):
if back_handler:
back_handler()
return
elif opt == "pair":
pairing_menu(back_handler=back_handler)
return
elif opt == "account":
account_menu(back_handler=back_handler)
return
elif opt == "integrations":
integrations_menu(back_handler=back_handler)
return
elif opt == "dataset":
dataset_menu(back_handler=back_handler)
return
elif opt == "selene":
CONFIGURATION["selene"]["enabled"] = not CONFIGURATION["selene"]["enabled"]
CONFIGURATION.store()
selene_menu(back_handler=back_handler)

2
requirements.txt
View file

@ -1,3 +1,3 @@
pywebio pywebio
ovos-local-backend>=0.1.5a2 ovos-backend-client
cutecharts cutecharts