Initial config addition

app.py

@@ -16,6 +16,7 @@
     CognitionImport,
     CognitionPrepareProject,
 )
+from config_handler import SERVICES_TO_NOTIFY, init_config
 from fast_api.routes.organization import router as org_router
 from fast_api.routes.project import router as project_router
 from fast_api.routes.project_setting import router as project_setting_router
@@ -40,6 +41,7 @@
 from starlette.routing import Route, Mount
 
 from controller.project.manager import check_in_deletion_projects
+from notify_handler import notify_others_about_change_thread
 from route_prefix import (
     PREFIX_ORGANIZATION,
     PREFIX_PROJECT,
@@ -154,5 +156,8 @@
 clean_up.clean_up_database()
 clean_up.clean_up_disk()
 
+init_config()
+notify_others_about_change_thread(SERVICES_TO_NOTIFY)
+
 session.start_session_cleanup_thread()
 log_storage.start_persist_thread()

base_config.json

@@ -0,0 +1,14 @@
+{
+    "is_managed": null,
+    "is_demo": null,
+    "allow_data_tracking": true,
+    "s3_region": null,
+    "spacy_downloads": [
+        "en_core_web_sm",
+        "de_core_news_sm"
+    ],
+    "tokens": {
+        "INTERCOM": ""
+    },
+    "KERN_S3_ENDPOINT": null
+}

config_handler.py

@@ -0,0 +1,98 @@
+from typing import Dict, Any
+import os
+import json
+
+__blacklist_base_config = ["is_managed", "is_demo"]
+__config = None
+
+BASE_CONFIG_PATH = "base_config.json"
+CURRENT_CONFIG_PATH = "/config/current_config.json"
+
+SERVICES_TO_NOTIFY = {
+    "EMBEDDER": "http://refinery-embedder:80",
+    "UPDATER": "http://refinery-updater:80",
+    "TOKENIZER": "http://refinery-tokenizer:80",
+}
+
+
+def __read_and_change_base_config():
+    print("reading base config file", flush=True)
+    global __config
+    f = open(BASE_CONFIG_PATH)
+    __config = json.load(f)
+
+    print("transfer os variables", flush=True)
+    __config["is_managed"] = os.getenv("IS_MANAGED") == "1"
+    __config["is_demo"] = os.getenv("IS_DEMO") == "1"
+    __config["tokens"]["INTERCOM"] = os.getenv("INTERCOM", "")
+    __config["s3_region"] = os.getenv("S3_REGION", "eu-west-1")
+
+    __save_current_config()
+
+
+def change_config(changes: Dict[str, Any]) -> bool:
+    global __config
+    something_changed = False
+    for key in changes:
+        if key == "KERN_S3_ENDPOINT":
+            continue
+        if key in __config:
+            if isinstance(changes[key], dict):
+                for subkey in changes[key]:
+                    if subkey in __config[key]:
+                        __config[key][subkey] = changes[key][subkey]
+                        something_changed = True
+            else:
+                __config[key] = changes[key]
+                something_changed = True
+    if something_changed:
+        __save_current_config()
+    else:
+        print("nothing was changed with input", changes, flush=True)
+    return something_changed
+
+
+def __save_current_config() -> None:
+    print("saving config file", flush=True)
+    with open(CURRENT_CONFIG_PATH, "w") as f:
+        json.dump(__config, f, indent=4)
+
+
+def init_config() -> None:
+    if not os.path.exists(CURRENT_CONFIG_PATH):
+        __read_and_change_base_config()
+    else:
+        __load_and_remove_outdated_config_keys()
+    # this one is to be set on every start to ensure its up to date
+    print("setting s3 endpoint", flush=True)
+    __config["KERN_S3_ENDPOINT"] = os.getenv("KERN_S3_ENDPOINT")
+
+
+def __load_and_remove_outdated_config_keys():
+    if not os.path.exists(CURRENT_CONFIG_PATH):
+        return
+
+    global __config
+    with open(CURRENT_CONFIG_PATH) as f:
+        __config = json.load(f)
+
+    with open(BASE_CONFIG_PATH) as f:
+        base_config = json.load(f)
+
+    to_remove = [key for key in __config if key not in base_config]
+
+    if len(to_remove) > 0:
+        print("removing outdated config keys", to_remove, flush=True)
+        for key in to_remove:
+            del __config[key]
+        __save_current_config()
+
+
+def get_config(basic: bool = True) -> Dict[str, Any]:
+    global __config, __blacklist_base_config
+    if not basic:
+        return __config
+
+    return {
+        key: __config[key] for key in __config if key not in __blacklist_base_config
+    }

fast_api/models.py

@@ -448,3 +448,7 @@ class UpdateCustomerButton(BaseModel):
     location: Optional[CustomerButtonLocation] = None
     visible: Optional[StrictBool] = None
     config: Optional[Dict[StrictStr, Any]] = None
+
+
+class ChangeRequest(BaseModel):
+    dict_string: str

fast_api/routes/misc.py

@@ -1,13 +1,15 @@
 import json
-from fastapi import APIRouter, Body, Request, status
+from fastapi import APIRouter, Body, Request, status, responses
 from fastapi.responses import PlainTextResponse
+from config_handler import SERVICES_TO_NOTIFY, change_config, get_config
 from exceptions.exceptions import ProjectAccessError
 from fast_api.models import (
     CancelTaskBody,
     ModelProviderDeleteModelBody,
     ModelProviderDownloadModelBody,
     CreateCustomerButton,
     UpdateCustomerButton,
+    ChangeRequest,
 )
 from fast_api.routes.client_response import pack_json_result, SILENT_SUCCESS_RESPONSE
 from typing import Dict, Optional
@@ -17,6 +19,7 @@
 from controller.monitor import manager as controller_manager
 from controller.model_provider import manager as model_provider_manager
 from controller.task_master import manager as task_master_manager
+from notify_handler import notify_others_about_change_thread
 from submodules.model import enums
 from submodules.model.global_objects import customer_button as customer_button_db_go
 import util.user_activity
@@ -47,6 +50,27 @@ def get_is_demo(request: Request) -> Dict:
     return pack_json_result({"data": {"isDemo": is_demo}})
 
 
+@router.post("/change_config")
+def change(request: ChangeRequest) -> responses.PlainTextResponse:
+    if change_config(json.loads(request.dict_string)):
+        notify_others_about_change_thread(SERVICES_TO_NOTIFY)
+    return responses.PlainTextResponse(status_code=status.HTTP_200_OK)
+
+
+@router.get("/full_config")
+def full_config() -> responses.JSONResponse:
+    return responses.JSONResponse(
+        status_code=status.HTTP_200_OK, content=get_config(False)
+    )
+
+
+@router.get("/base_config")
+def base_config() -> responses.JSONResponse:
+    return responses.JSONResponse(
+        status_code=status.HTTP_200_OK, content=get_config(True)
+    )
+
+
 @router.get("/version-overview")
 def get_version_overview(request: Request) -> Dict:
     data = manager.get_version_overview()

notify_handler.py

@@ -0,0 +1,25 @@
+from typing import Dict
+
+import requests
+import traceback
+
+from submodules.model import daemon
+
+
+def notify_others_about_change(notify: Dict[str, str]) -> None:
+
+    for key in notify:
+        url = f"{notify[key]}/config_changed"
+        try:
+            response = requests.put(url)
+            if response.status_code != 200:
+                print(f"couldn't notify - code:{response.status_code}", key, flush=True)
+        except requests.exceptions.ConnectionError:
+            print("couldn't notify", key, flush=True)
+        except Exception:
+            print(traceback.format_exc(), flush=True)
+
+
+def notify_others_about_change_thread(notify: Dict[str, str]) -> None:
+    # needs to be threaded to work with circular requests (this notifies about changes -> service requests full config from this)
+    daemon.run_without_db_token(notify_others_about_change, notify)