diff --git a/helper/import_precise_community.py b/helper/import_precise_community.py index c2c9f13..0748b92 100644 --- a/helper/import_precise_community.py +++ b/helper/import_precise_community.py @@ -1,7 +1,7 @@ import os import random -from ovos_local_backend.configuration import CONFIGURATION +from ovos_backend_manager.configuration import CONFIGURATION from ovos_local_backend.database.metrics import JsonMetricDatabase, Metric from ovos_local_backend.database.wakewords import JsonWakeWordDatabase, WakeWordRecording from os.path import dirname diff --git a/ovos_backend_manager/app.py b/ovos_backend_manager/app.py index 7dd6119..107217d 100644 --- a/ovos_backend_manager/app.py +++ b/ovos_backend_manager/app.py @@ -3,7 +3,7 @@ import os import requests from flask import Flask, request from oauthlib.oauth2 import WebApplicationClient -from ovos_local_backend.database.oauth import OAuthTokenDatabase, OAuthApplicationDatabase +from ovos_backend_manager.configuration import DB from pywebio.platform.flask import webio_view from ovos_backend_manager.menu import start @@ -23,7 +23,7 @@ def oauth_callback(oauth_id): params = dict(request.args) code = params["code"] - data = OAuthApplicationDatabase()[oauth_id] + data = DB.get_oauth_app(oauth_id) client_id = data["client_id"] client_secret = data["client_secret"] token_endpoint = data["token_endpoint"] @@ -43,8 +43,7 @@ def oauth_callback(oauth_id): auth=(client_id, client_secret), ).json() - with OAuthTokenDatabase() as db: - db.add_token(oauth_id, token_response) + DB.add_oauth_token(oauth_id, token_response) return params diff --git a/ovos_backend_manager/backend.py b/ovos_backend_manager/backend.py index 9cbf02e..966973a 100644 --- a/ovos_backend_manager/backend.py +++ b/ovos_backend_manager/backend.py @@ -1,8 +1,8 @@ import json import os -from ovos_local_backend.configuration import CONFIGURATION -from ovos_local_backend.utils.geolocate import get_location_config +from ovos_backend_manager.configuration import CONFIGURATION +from ovos_backend_client.api import GeolocationApi from pywebio.input import textarea, select, actions from pywebio.output import put_table, put_markdown, popup, put_code, put_image, use_scope @@ -18,7 +18,6 @@ def backend_menu(back_handler=None): ['Device Authentication enabled', not CONFIGURATION["skip_auth"]], ['Location override enabled', CONFIGURATION["override_location"]], ['IP Geolocation enabled', CONFIGURATION["geolocate"]], - ['Selene Proxy enabled', CONFIGURATION["selene"]["enabled"]], ['Default TTS', CONFIGURATION["default_tts"]], ['Default Wake Word', CONFIGURATION["default_ww"]], ['Default date format', CONFIGURATION["date_format"]], @@ -76,7 +75,7 @@ def backend_menu(back_handler=None): loc = textarea("Enter an address", placeholder="Anywhere street Any city Nº234", required=True) - data = get_location_config(loc) + data = GeolocationApi().get_geolocation(loc) CONFIGURATION["default_location"] = data with popup(f"Default location set to: {loc}"): put_code(json.dumps(data, ensure_ascii=True, indent=2), "json") diff --git a/ovos_backend_manager/configuration.py b/ovos_backend_manager/configuration.py new file mode 100644 index 0000000..fb0758c --- /dev/null +++ b/ovos_backend_manager/configuration.py @@ -0,0 +1,215 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from os.path import exists, expanduser + +from json_database import JsonConfigXDG +from ovos_utils.log import LOG +from ovos_backend_client.api import DatabaseApi + + +DEFAULT_CONFIG = { + "lang": "en-us", # default language + "database": "sqlite:///ovos_backend.db", + "stt": { + "module": "ovos-stt-plugin-server", + "ovos-stt-plugin-server": { + "url": "https://stt.openvoiceos.com/stt" + } + }, + "backend_port": 6712, + "admin_key": "", # To enable simply set this string to something + "skip_auth": False, # you almost certainly do not want this, only for atypical use cases such as ovos-qubes + "default_location": { + "city": { + "code": "Lawrence", + "name": "Lawrence", + "state": { + "code": "KS", + "name": "Kansas", + "country": { + "code": "US", + "name": "United States" + } + } + }, + "coordinate": { + "latitude": 38.971669, + "longitude": -95.23525 + }, + "timezone": { + "code": "America/Chicago", + "name": "Central Standard Time", + "dstOffset": 3600000, + "offset": -21600000 + } + }, + "default_ww": "hey_mycroft", # needs to be present below + "ww_configs": { # these can be exposed in a web UI for selection + "android": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/android.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "computer": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/computer.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "hey_chatterbox": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_chatterbox.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "hey_firefox": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_firefox.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "hey_k9": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_k9.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "hey_kit": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_kit.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "hey_moxie": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_moxie.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "hey_mycroft": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_mycroft.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "hey_scout": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/hey_scout.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "marvin": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/marvin.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "o_sauro": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/o_sauro.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "sheila": {"module": "ovos-ww-plugin-precise-lite", + "model": "https://github.com/OpenVoiceOS/precise-lite-models/raw/master/wakewords/en/sheila.tflite", + "expected_duration": 3, + "trigger_level": 3, + "sensitivity": 0.5 + }, + "hey_jarvis": {"module": "ovos-ww-plugin-vosk", + "rule": "fuzzy", + "samples": [ + "hay jarvis", + "hey jarvis", + "hay jarbis", + "hey jarbis" + ] + }, + "christopher": {"module": "ovos-ww-plugin-vosk", + "rule": "fuzzy", + "samples": [ + "christopher" + ] + }, + "hey_ezra": {"module": "ovos-ww-plugin-vosk", + "rule": "fuzzy", + "samples": [ + "hay ezra", + "hey ezra" + ] + }, + "hey_ziggy": {"module": "ovos-ww-plugin-vosk", + "rule": "fuzzy", + "samples": [ + "hey ziggy", + "hay ziggy" + ] + }, + "hey_neon": {"module": "ovos-ww-plugin-vosk", + "rule": "fuzzy", + "samples": [ + "hey neon", + "hay neon" + ] + } + }, + "default_tts": "American Male", # needs to be present below + "tts_configs": { # these can be exposed in a web UI for selection + "American Male": {"module": "ovos-tts-plugin-mimic2", "voice": "kusal"}, + "British Male": {"module": "ovos-tts-plugin-mimic", "voice": "ap"} + }, + "date_format": "DMY", + "system_unit": "metric", + "time_format": "full", + "geolocate": True, + "override_location": False, + "api_version": "v1", + "data_path": expanduser("~"), + "record_utterances": False, + "record_wakewords": False, + "microservices": { + # if query fail, attempt to use free ovos services + "ovos_fallback": True, + # backend can be auto/local/ovos + # auto == attempt local -> ovos + "wolfram_provider": "auto", + "weather_provider": "auto", + # auto == OpenStreetMap default + # valid - osm/arcgis/geocode_farm + "geolocation_provider": "auto", + # secret keys + "wolfram_key": "", + "owm_key": "" + }, + "email": { + "username": None, + "password": None + } +} + +CONFIGURATION = JsonConfigXDG("ovos_backend") +if not exists(CONFIGURATION.path): + CONFIGURATION.merge(DEFAULT_CONFIG, skip_empty=False) + CONFIGURATION.store() + LOG.info(f"Saved default configuration: {CONFIGURATION.path}") +else: + # set any new default values since file creation + for k, v in DEFAULT_CONFIG.items(): + if k not in CONFIGURATION: + CONFIGURATION[k] = v + LOG.info(f"Loaded configuration: {CONFIGURATION.path}") + + +DB = DatabaseApi(CONFIGURATION["admin_key"]) + diff --git a/ovos_backend_manager/datasets.py b/ovos_backend_manager/datasets.py index b70252e..964d78d 100644 --- a/ovos_backend_manager/datasets.py +++ b/ovos_backend_manager/datasets.py @@ -3,28 +3,25 @@ import os import time from base64 import b64encode -from ovos_local_backend.configuration import CONFIGURATION -from ovos_local_backend.database.settings import DeviceDatabase -from ovos_local_backend.database.utterances import JsonUtteranceDatabase -from ovos_local_backend.database.wakewords import JsonWakeWordDatabase from pywebio.input import actions, file_upload, input_group, textarea, select from pywebio.output import put_text, put_code, use_scope, put_markdown, popup, put_image, put_file, put_html, \ put_buttons, put_table +from ovos_backend_manager.configuration import CONFIGURATION, DB -def _render_ww(idx, db=None): - db = db or JsonWakeWordDatabase() + +def _render_ww(rec_id): def on_tag(bt): data["tag"] = bt - db[idx]["tag"] = bt - db.commit() - _render_ww(idx, db) + DB.update_ww_recording(rec_id, tag=bt) + _render_ww(rec_id) with use_scope("main_view", clear=True): - data = db[idx] # id == db_position + 1 + data = DB.get_ww_recording(rec_id) data["tag"] = data.get("tag") or "untagged" + # TODO - get binary_data directly if os.path.isfile(data["path"]): content = open(data["path"], 'rb').read() html = f""" @@ -50,72 +47,73 @@ def _render_ww(idx, db=None): def ww_select(back_handler=None, uuid=None, ww=None): buttons = [] - db = JsonWakeWordDatabase() - if not len(db): + if not len(DB.list_ww_recordings()): with use_scope("main_view", clear=True): put_text("No wake words uploaded yet!") datasets_menu(back_handler=back_handler) return - for m in db: + for m in DB.list_ww_recordings(): if uuid is not None and m["uuid"] != uuid: continue if ww is not None and m["transcription"] != ww: continue - name = f"{m['wakeword_id']}-{m['transcription']}" - buttons.append({'label': name, 'value': m['wakeword_id']}) + name = f"{m['recording_id']}-{m['transcription']}" + buttons.append({'label': name, 'value': m['recording_id']}) if len(buttons) == 0: with use_scope("main_view", clear=True): put_text("No wake words uploaded from this device yet!") - opt = "main" - else: - if back_handler: - buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) - opt = actions(label="Select a WakeWord recording", - buttons=buttons) - if opt == "main": ww_menu(back_handler=back_handler) return - _render_ww(opt - 1, db) + elif back_handler: + buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) - ww_select(back_handler=back_handler, ww=ww, uuid=uuid) + rec_id = actions(label="Select a WakeWord recording", buttons=buttons) + + if rec_id == "main": # go back + ww_menu(back_handler=back_handler) + else: + _render_ww(rec_id) + ww_select(back_handler=back_handler, ww=ww, uuid=uuid) def utt_select(back_handler=None, uuid=None, utt=None): buttons = [] - db = JsonUtteranceDatabase() - if not len(db): + if not len(DB.list_stt_recordings()): with use_scope("main_view", clear=True): put_text("No utterances uploaded yet!") datasets_menu(back_handler=back_handler) return - for m in db: + for m in DB.list_stt_recordings(): if uuid is not None and m["uuid"] != uuid: continue if utt is not None and m["transcription"] != utt: continue - name = f"{m['utterance_id']}-{m['transcription']}" - buttons.append({'label': name, 'value': m['utterance_id']}) + name = f"{m['recording_id']}-{m['transcription']}" + buttons.append({'label': name, 'value': m['recording_id']}) if len(buttons) == 0: with use_scope("main_view", clear=True): put_text("No utterances uploaded from this device yet!") - opt = "main" + utt_menu(back_handler=back_handler) + return else: if back_handler: buttons.insert(0, {'label': '<- Go Back', 'value': "main"}) - opt = actions(label="Select a Utterance recording", - buttons=buttons) - if opt == "main": - utt_menu(back_handler=back_handler) - return + rec_id = actions(label="Select a Utterance recording", + buttons=buttons) + if rec_id == "main": + utt_menu(back_handler=back_handler) + return with use_scope("main_view", clear=True): - data = db[opt - 1] # id == db_position + 1 + # opt is recording_id + data = DB.get_stt_recording(rec_id) put_code(json.dumps(data, indent=4), "json") + # TODO - get binary data from api if os.path.isfile(data["path"]): content = open(data["path"], 'rb').read() html = f"""