Merge main into stable/4.0.x. Update 20231018

* commit '2d8d4a4f081c7d11401b7dd3f79363fb8bad2cd7': Make `THREAD_WATCH_TIMEOUT` configurable via env var (#1903) Topic json upload rework (#1914) Make user.forget_password two-email test safer (#1918) I1896 account.import (#1912) Make meeting_user actions backend internal (#1895)
OpenSlides · Oct 18, 2023 · dc894be · dc894be
2 parents a3d2cd1 + 2d8d4a4
commit dc894be
Show file tree

Hide file tree

Showing 48 changed files with 3,298 additions and 1,965 deletions.
diff --git a/README.md b/README.md
@@ -71,77 +71,90 @@ The action component listens to port 9002. The presenter component listens to po
 
 ## Environment variables
 
-* OPENSLIDES_BACKEND_COMPONENT
+### Functionality
+
+* `OPENSLIDES_BACKEND_COMPONENT`
 
   Use one of the following values to start only one component of this service: `action` or `presenter`. Defaults to all of them using different child processes. If using `all` you can shut down all compontes by sending SIGTERM to Python master process.
 
-* ACTION_PORT
+* `ACTION_PORT`
 
-  Action component listens on this port. Default: 9002
+  Action component listens on this port. Default: `9002`
 
-* PRESENTER_PORT
+* `PRESENTER_PORT`
 
-  Presenter component listens on this port. Default 9003
+  Presenter component listens on this port. Default `9003`
 
-* OPENSLIDES_DEVELOPMENT
+* `OPENTELEMETRY_ENABLED`
 
-  Set this variable e. g. to 1 to set loglevel to `debug` and activate Gunicorn's reload mechanism.
+  Set this variable e. g. to `1` to enable span reporting to an OpenTelemetry collector (defined in the main OpenSlides repository).
 
-* OPENTELEMETRY_ENABLED
+* `OPENSLIDES_LOGLEVEL`
 
-  Set this variable e. g. to 1 to enable span reporting to an OpenTelemetry collector (defined in the main OpenSlides repository).
+  In production mode you can set the loglevel to `debug`, `info`, `warning`, `error` or `critical`. Default is `info`.
 
-* OPENSLIDES_LOGLEVEL
+* `OPENSLIDES_BACKEND_NUM_WORKERS`
 
-  In production mode you can set the loglevel to `debug`, `info`, `warning`, `error` or `critical`. Default is `info`.
+  Number of Gunicorn workers. Default: `1`
 
-* OPENSLIDES_BACKEND_RAISE_4XX
+* `OPENSLIDES_BACKEND_WORKER_TIMEOUT`
 
-  Set this variable to raise HTTP 400 and 403 as exceptions instead of valid HTTP responses.
+  Gunicorn worker timeout in seconds. Default: `30`
+
+* `OPENSLIDES_BACKEND_THREAD_WATCH_TIMEOUT`
 
-* DATASTORE_READER_PROTOCOL
+  Seconds after which an action is delegated to an action worker. `-1` deactivates action workers all together. Default: `1.0`
 
-  Protocol of datastore reader service. Default: http
+### Development
+
+* `OPENSLIDES_DEVELOPMENT`
+
+  Set this variable e. g. to `1` to set loglevel to `debug` and activate Gunicorn's reload mechanism.
+
+* `OPENSLIDES_BACKEND_RAISE_4XX`
+
+  Set this variable to raise HTTP 400 and 403 as exceptions instead of valid HTTP responses.
 
-* DATASTORE_READER_HOST
+### Connection to other services
+* `DATASTORE_READER_PROTOCOL`
 
-  Host of datastore reader service. Default: localhost
+  Protocol of datastore reader service. Default: `http`
 
-* DATASTORE_READER_PORT
+* `DATASTORE_READER_HOST`
 
-  Port of datastore reader service. Default: 9010
+  Host of datastore reader service. Default: `localhost`
 
-* DATASTORE_READER_PATH
+* `DATASTORE_READER_PORT`
 
-  Path of datastore reader service. Default: /internal/datastore/reader
+  Port of datastore reader service. Default: `9010`
 
-* DATASTORE_WRITER_PROTOCOL
+* `DATASTORE_READER_PATH`
 
-  Protocol of datastore writer service. Default: http
+  Path of datastore reader service. Default: `/internal/datastore/reader`
 
-* DATASTORE_WRITER_HOST
+* `DATASTORE_WRITER_PROTOCOL`
 
-  Host of datastore writer service. Default: localhost
+  Protocol of datastore writer service. Default: `http`
 
-* DATASTORE_WRITER_PORT
+* `DATASTORE_WRITER_HOST`
 
-  Port of datastore writer service. Default: 9011
+  Host of datastore writer service. Default: `localhost`
 
-* DATASTORE_WRITER_PATH
+* `DATASTORE_WRITER_PORT`
 
-  Path of datastore writer service. Default: /internal/datastore/writer
+  Port of datastore writer service. Default: `9011`
 
-* OPENSLIDES_BACKEND_NUM_WORKERS
+* `DATASTORE_WRITER_PATH`
 
-  Number of Gunicorn workers. Default: 1
+  Path of datastore writer service. Default: `/internal/datastore/writer`
 
-* OPENSLIDES_BACKEND_WORKER_TIMEOUT
+* `AUTH_HOST`
 
-  Gunicorn worker timeout in seconds. Default: 30
+  Host of auth service. Used by the `authlib` package. Default: `localhost`
 
-* AUTH_HOST and AUTH_PORT
+* `AUTH_PORT`
 
-  Implicitly used by the authlib to get the endpoint for the auth-service
+  Port of auth service. Used by the `authlib` package. Default: `9004`
 
 
 # Some curl examples

diff --git a/global/meta/models.yml b/global/meta/models.yml
@@ -3837,3 +3837,27 @@ action_worker:
   result:
     type: JSON
     restriction_mode: A
+
+import_preview:
+  id:
+    type: number
+    restriction_mode: A
+  name:
+    type: string
+    required: true
+    restriction_mode: A
+  state:
+    type: string
+    required: true
+    enum:
+      - warning
+      - error
+      - done
+    restriction_mode: A
+  created:
+    type: timestamp
+    required: true
+    restriction_mode: A
+  result:
+    type: JSON
+    restriction_mode: A
diff --git a/openslides_backend/action/action_worker.py b/openslides_backend/action/action_worker.py
@@ -18,8 +18,6 @@
 from .action_handler import ActionHandler
 from .util.typing import ActionsResponse, Payload
 
-THREAD_WATCH_TIMEOUT = 1.0
-
 
 def handle_action_in_worker_thread(
     payload: Payload,
@@ -51,7 +49,9 @@ def handle_action_in_worker_thread(
     action_worker_thread.start()
     while not action_worker_thread.started:
         sleep(0.001)  # The action_worker_thread should gain the lock and NOT this one
-    if lock.acquire(timeout=THREAD_WATCH_TIMEOUT):
+
+    timeout = float(handler.env.OPENSLIDES_BACKEND_THREAD_WATCH_TIMEOUT)
+    if lock.acquire(timeout=timeout):
         lock.release()
         if hasattr(action_worker_thread, "exception"):
             raise action_worker_thread.exception
@@ -106,7 +106,7 @@ def initial_action_worker_write(self) -> str:
                 self.new_id = self.datastore.reserve_id(collection="action_worker")
                 self.fqid = fqid_from_collection_and_id("action_worker", self.new_id)
             try:
-                self.datastore.write_action_worker(
+                self.datastore.write_without_events(
                     WriteRequest(
                         events=[
                             Event(
@@ -138,7 +138,7 @@ def initial_action_worker_write(self) -> str:
     def continue_action_worker_write(self) -> None:
         current_time = round(time())
         with self.datastore.get_database_context():
-            self.datastore.write_action_worker(
+            self.datastore.write_without_events(
                 WriteRequest(
                     events=[
                         Event(
@@ -190,7 +190,7 @@ def final_action_worker_write(self, action_worker_thread: "ActionWorker") -> Non
                     f"aborted action_worker '{self.fqid}' ({self.action_names}) {current_time}: {message}"
                 )
 
-            self.datastore.write_action_worker(
+            self.datastore.write_without_events(
                 WriteRequest(
                     events=[
                         Event(

diff --git a/openslides_backend/action/actions/agenda_item/agenda_creation.py b/openslides_backend/action/actions/agenda_item/agenda_creation.py
@@ -82,18 +82,21 @@ def check_dependant_action_execution_agenda_item(
     def get_dependent_action_data_agenda_item(
         self, instance: Dict[str, Any], CreateActionClass: Type[Action]
     ) -> List[Dict[str, Any]]:
-        agenda_item_action_data = {
-            "content_object_id": fqid_from_collection_and_id(
-                self.model.collection, instance["id"]
-            ),
-        }
-        for extra_field in agenda_creation_properties.keys():
-            if extra_field == f"{AGENDA_PREFIX}create":
-                # This field should not be provided to the AgendaItemCreate action.
-                continue
-            prefix_len = len(AGENDA_PREFIX)
-            extra_field_without_prefix = extra_field[prefix_len:]
-            value = instance.pop(extra_field, None)
-            if value is not None:
-                agenda_item_action_data[extra_field_without_prefix] = value
+        agenda_item_action_data = self.remove_agenda_prefix_from_fieldnames(instance)
+        agenda_item_action_data["content_object_id"] = fqid_from_collection_and_id(
+            self.model.collection, instance["id"]
+        )
         return [agenda_item_action_data]
+
+    @staticmethod
+    def remove_agenda_prefix_from_fieldnames(
+        instance: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        prefix_len = len(AGENDA_PREFIX)
+        extra_field = f"{AGENDA_PREFIX}create"  # This field should not be provided to the AgendaItemCreate action.
+        agenda_item = {
+            field[prefix_len:]: value
+            for field in agenda_creation_properties.keys()
+            if field != extra_field and (value := instance.pop(field, None)) is not None
+        }
+        return agenda_item
diff --git a/openslides_backend/action/actions/meeting/import_.py b/openslides_backend/action/actions/meeting/import_.py
@@ -153,6 +153,7 @@ def remove_not_allowed_fields(self, instance: Dict[str, Any]) -> None:
             user.pop("committee_management_ids", None)
         self.get_meeting_from_json(json_data).pop("organization_tag_ids", None)
         json_data.pop("action_worker", None)
+        json_data.pop("import_preview", None)
 
     def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         meeting_json = instance["meeting"]

diff --git a/openslides_backend/action/actions/meeting_user/create.py b/openslides_backend/action/actions/meeting_user/create.py
@@ -5,16 +5,17 @@
 from openslides_backend.shared.typing import HistoryInformation
 
 from ....models.models import MeetingUser
-from ....permissions.permissions import Permissions
 from ...generics.create import CreateAction
 from ...mixins.meeting_user_helper import get_meeting_user_filter
+from ...util.action_type import ActionType
 from ...util.default_schema import DefaultSchema
 from ...util.register import register_action
-from .mixin import MeetingUserMixin
+from .history_mixin import MeetingUserHistoryMixin
+from .mixin import meeting_user_standard_fields
 
 
-@register_action("meeting_user.create")
-class MeetingUserCreate(MeetingUserMixin, CreateAction):
+@register_action("meeting_user.create", action_type=ActionType.BACKEND_INTERNAL)
+class MeetingUserCreate(MeetingUserHistoryMixin, CreateAction):
     """
     Action to create a meeting user.
     """
@@ -25,10 +26,9 @@ class MeetingUserCreate(MeetingUserMixin, CreateAction):
         optional_properties=[
             "about_me",
             "group_ids",
-            *MeetingUserMixin.standard_fields,
+            *meeting_user_standard_fields,
         ],
     )
-    permission = Permissions.User.CAN_MANAGE
 
     def update_instance(self, instance: Dict[str, Any]) -> Dict[str, Any]:
         if self.datastore.exists(

diff --git a/openslides_backend/action/actions/meeting_user/delete.py b/openslides_backend/action/actions/meeting_user/delete.py
@@ -4,21 +4,20 @@
 from openslides_backend.shared.typing import HistoryInformation
 
 from ....models.models import MeetingUser
-from ....permissions.permissions import Permissions
 from ...generics.delete import DeleteAction
+from ...util.action_type import ActionType
 from ...util.default_schema import DefaultSchema
 from ...util.register import register_action
 
 
-@register_action("meeting_user.delete")
+@register_action("meeting_user.delete", action_type=ActionType.BACKEND_INTERNAL)
 class MeetingUserDelete(DeleteAction):
     """
     Action to delete a meeting user.
     """
 
     model = MeetingUser()
     schema = DefaultSchema(MeetingUser()).get_delete_schema()
-    permission = Permissions.User.CAN_MANAGE
 
     def get_history_information(self) -> Optional[HistoryInformation]:
         users = self.get_instances_with_fields(["user_id", "meeting_id"])