From 8253c0cd6a97d4b2f259de944d7153038bea8ae8 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 5 Nov 2024 14:42:50 +0100 Subject: [PATCH] Initial config addition --- app.py | 5 +++ base_config.json | 14 ++++++ config/.gitkeep | 0 config_handler.py | 98 +++++++++++++++++++++++++++++++++++++++++ fast_api/models.py | 4 ++ fast_api/routes/misc.py | 26 ++++++++++- notify_handler.py | 25 +++++++++++ 7 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 base_config.json create mode 100644 config/.gitkeep create mode 100644 config_handler.py create mode 100644 notify_handler.py diff --git a/app.py b/app.py index fb2ebc69..7dc01c4e 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,7 @@ CognitionImport, CognitionPrepareProject, ) +from config_handler import SERVICES_TO_NOTIFY, init_config from fast_api.routes.organization import router as org_router from fast_api.routes.project import router as project_router from fast_api.routes.project_setting import router as project_setting_router @@ -40,6 +41,7 @@ from starlette.routing import Route, Mount from controller.project.manager import check_in_deletion_projects +from notify_handler import notify_others_about_change_thread from route_prefix import ( PREFIX_ORGANIZATION, PREFIX_PROJECT, @@ -154,5 +156,8 @@ clean_up.clean_up_database() clean_up.clean_up_disk() +init_config() +notify_others_about_change_thread(SERVICES_TO_NOTIFY) + session.start_session_cleanup_thread() log_storage.start_persist_thread() diff --git a/base_config.json b/base_config.json new file mode 100644 index 00000000..b82e8f59 --- /dev/null +++ b/base_config.json @@ -0,0 +1,14 @@ +{ + "is_managed": null, + "is_demo": null, + "allow_data_tracking": true, + "s3_region": null, + "spacy_downloads": [ + "en_core_web_sm", + "de_core_news_sm" + ], + "tokens": { + "INTERCOM": "" + }, + "KERN_S3_ENDPOINT": null +} \ No newline at end of file diff --git a/config/.gitkeep b/config/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/config_handler.py b/config_handler.py new file mode 100644 index 00000000..c49e61c7 --- /dev/null +++ b/config_handler.py @@ -0,0 +1,98 @@ +from typing import Dict, Any +import os +import json + +__blacklist_base_config = ["is_managed", "is_demo"] +__config = None + +BASE_CONFIG_PATH = "base_config.json" +CURRENT_CONFIG_PATH = "/config/current_config.json" + +SERVICES_TO_NOTIFY = { + "EMBEDDER": "http://refinery-embedder:80", + "UPDATER": "http://refinery-updater:80", + "TOKENIZER": "http://refinery-tokenizer:80", +} + + +def __read_and_change_base_config(): + print("reading base config file", flush=True) + global __config + f = open(BASE_CONFIG_PATH) + __config = json.load(f) + + print("transfer os variables", flush=True) + __config["is_managed"] = os.getenv("IS_MANAGED") == "1" + __config["is_demo"] = os.getenv("IS_DEMO") == "1" + __config["tokens"]["INTERCOM"] = os.getenv("INTERCOM", "") + __config["s3_region"] = os.getenv("S3_REGION", "eu-west-1") + + __save_current_config() + + +def change_config(changes: Dict[str, Any]) -> bool: + global __config + something_changed = False + for key in changes: + if key == "KERN_S3_ENDPOINT": + continue + if key in __config: + if isinstance(changes[key], dict): + for subkey in changes[key]: + if subkey in __config[key]: + __config[key][subkey] = changes[key][subkey] + something_changed = True + else: + __config[key] = changes[key] + something_changed = True + if something_changed: + __save_current_config() + else: + print("nothing was changed with input", changes, flush=True) + return something_changed + + +def __save_current_config() -> None: + print("saving config file", flush=True) + with open(CURRENT_CONFIG_PATH, "w") as f: + json.dump(__config, f, indent=4) + + +def init_config() -> None: + if not os.path.exists(CURRENT_CONFIG_PATH): + __read_and_change_base_config() + else: + __load_and_remove_outdated_config_keys() + # this one is to be set on every start to ensure its up to date + print("setting s3 endpoint", flush=True) + __config["KERN_S3_ENDPOINT"] = os.getenv("KERN_S3_ENDPOINT") + + +def __load_and_remove_outdated_config_keys(): + if not os.path.exists(CURRENT_CONFIG_PATH): + return + + global __config + with open(CURRENT_CONFIG_PATH) as f: + __config = json.load(f) + + with open(BASE_CONFIG_PATH) as f: + base_config = json.load(f) + + to_remove = [key for key in __config if key not in base_config] + + if len(to_remove) > 0: + print("removing outdated config keys", to_remove, flush=True) + for key in to_remove: + del __config[key] + __save_current_config() + + +def get_config(basic: bool = True) -> Dict[str, Any]: + global __config, __blacklist_base_config + if not basic: + return __config + + return { + key: __config[key] for key in __config if key not in __blacklist_base_config + } diff --git a/fast_api/models.py b/fast_api/models.py index bbd54529..b7a72e56 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -448,3 +448,7 @@ class UpdateCustomerButton(BaseModel): location: Optional[CustomerButtonLocation] = None visible: Optional[StrictBool] = None config: Optional[Dict[StrictStr, Any]] = None + + +class ChangeRequest(BaseModel): + dict_string: str diff --git a/fast_api/routes/misc.py b/fast_api/routes/misc.py index d503c25f..34d56b3f 100644 --- a/fast_api/routes/misc.py +++ b/fast_api/routes/misc.py @@ -1,6 +1,7 @@ import json -from fastapi import APIRouter, Body, Request, status +from fastapi import APIRouter, Body, Request, status, responses from fastapi.responses import PlainTextResponse +from config_handler import SERVICES_TO_NOTIFY, change_config, get_config from exceptions.exceptions import ProjectAccessError from fast_api.models import ( CancelTaskBody, @@ -8,6 +9,7 @@ ModelProviderDownloadModelBody, CreateCustomerButton, UpdateCustomerButton, + ChangeRequest, ) from fast_api.routes.client_response import pack_json_result, SILENT_SUCCESS_RESPONSE from typing import Dict, Optional @@ -17,6 +19,7 @@ from controller.monitor import manager as controller_manager from controller.model_provider import manager as model_provider_manager from controller.task_master import manager as task_master_manager +from notify_handler import notify_others_about_change_thread from submodules.model import enums from submodules.model.global_objects import customer_button as customer_button_db_go import util.user_activity @@ -47,6 +50,27 @@ def get_is_demo(request: Request) -> Dict: return pack_json_result({"data": {"isDemo": is_demo}}) +@router.post("/change_config") +def change(request: ChangeRequest) -> responses.PlainTextResponse: + if change_config(json.loads(request.dict_string)): + notify_others_about_change_thread(SERVICES_TO_NOTIFY) + return responses.PlainTextResponse(status_code=status.HTTP_200_OK) + + +@router.get("/full_config") +def full_config() -> responses.JSONResponse: + return responses.JSONResponse( + status_code=status.HTTP_200_OK, content=get_config(False) + ) + + +@router.get("/base_config") +def base_config() -> responses.JSONResponse: + return responses.JSONResponse( + status_code=status.HTTP_200_OK, content=get_config(True) + ) + + @router.get("/version-overview") def get_version_overview(request: Request) -> Dict: data = manager.get_version_overview() diff --git a/notify_handler.py b/notify_handler.py new file mode 100644 index 00000000..64064f2e --- /dev/null +++ b/notify_handler.py @@ -0,0 +1,25 @@ +from typing import Dict + +import requests +import traceback + +from submodules.model import daemon + + +def notify_others_about_change(notify: Dict[str, str]) -> None: + + for key in notify: + url = f"{notify[key]}/config_changed" + try: + response = requests.put(url) + if response.status_code != 200: + print(f"couldn't notify - code:{response.status_code}", key, flush=True) + except requests.exceptions.ConnectionError: + print("couldn't notify", key, flush=True) + except Exception: + print(traceback.format_exc(), flush=True) + + +def notify_others_about_change_thread(notify: Dict[str, str]) -> None: + # needs to be threaded to work with circular requests (this notifies about changes -> service requests full config from this) + daemon.run_without_db_token(notify_others_about_change, notify)