Merge staging/4.1.3 into stable/4.1.x. Update 20240312

* commit '7e10e3f80bc28f6a87cbb7ff426e84dd5fd3ab4f': (90 commits)
  Add cascade deletion to new motion fields (#2243) (#2284)
  Update all translations (#2282)
  Implement certain preventions for changing active speakers state (#2272)
  Update meta (#2269)
  Implement cascade delete for amendments (#2253)
  Automatically create PRs for commits on staging branches (#2256)
  Add staging branches to CI (#2255)
  Fix PoO for others (#2242)
  Make SLLoS ignore point-of-order time (#2240)
  Bump types-bleach in /requirements/partial (#2238)
  Update meta repository (#2235)
  Implement motion import (#1923)
  Enhance speaker creation (#2231)
  Add migration command to clear collectionfield tables (#2217)
  Allow initial_time to be changed regardless of speach status (#2220)
  Bump types-simplejson in /requirements/partial (#2223)
  Bump types-redis in /requirements/partial (#2221)
  Bump pypdf[crypto] from 4.0.1 to 4.0.2 in /requirements/partial (#2222)
  Bump pytest from 8.0.0 to 8.0.1 in /requirements/partial (#2224)
  Bump types-requests in /requirements/partial (#2225)
  ...

.github/workflows/continuous_integration.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - 'feature/**'
+      - 'staging/**'
   merge_group:
     types: [checks_requested]
 
@@ -100,6 +101,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
 
     - name: Set up Python
       uses: actions/setup-python@v5
@@ -111,11 +114,11 @@ jobs:
 
     - name: Check black
       if: always()
-      run: black --check --diff openslides_backend/ tests/ cli/
+      run: make check-black
 
     - name: Check isort
       if: always()
-      run: isort --check-only --diff openslides_backend/ tests/ cli/
+      run: make check-isort
 
     - name: Check flake8
       if: always()
@@ -125,6 +128,10 @@ jobs:
       if: always()
       run: make mypy
 
+    - name: Check pyupgrade
+      if: always()
+      run: make pyupgrade
+
   check-models-all:
     name: Check from models.yml up to permissions and jsons
     runs-on: ubuntu-latest
@@ -133,6 +140,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        submodules: true
 
     - name: Set up Python
       uses: actions/setup-python@v5
@@ -142,10 +151,6 @@ jobs:
     - name: Install requirements
       run: . requirements/export_service_commits.sh && pip install --requirement requirements/requirements_development.txt
 
-    - name: Validate models.yml
-      if: always()
-      run: make validate-models-yml
-
     - name: Check for up to date models.py
       if: always()
       run: make check-models

.github/workflows/staging-to-main.yml

@@ -0,0 +1,54 @@
+name: Copy staging commits to main
+
+on:
+  push:
+    branches:
+      - 'staging/4*'
+
+
+jobs:
+  create-pr-for-main:
+    name: Create PR against main branch
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout main
+      uses: actions/checkout@v4
+      with:
+        ref: main
+
+    - name: Set git credentials
+      run: |
+        git config --global user.name openslides-automation
+        git config --global user.email [email protected]
+    
+    - name: Cherry-pick new commit
+      id: cherry-pick
+      run: |
+        git fetch origin
+        git cherry-pick ${{ github.sha }} || {
+          echo "error=1" >> $GITHUB_OUTPUT
+          git add .
+          git cherry-pick --continue
+        }
+
+    - name: Generate access token
+      uses: tibdex/github-app-token@v2
+      id: generate-token
+      with:
+        app_id: ${{ secrets.AUTOMATION_APP_ID }}
+        private_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }}
+
+    - name: Create or update PR
+      uses: peter-evans/create-pull-request@v6
+      with:
+        token: ${{ steps.generate-token.outputs.token }}
+        commit-message: ${{ github.event.commits[0].message }}
+        branch: apply/commit-${{ github.sha }}
+        delete-branch: true
+        title: ${{ github.event.commits[0].message }}
+        body: "Triggered by commit [${{ github.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.sha }})\n\n${{ steps.cherry-pick.outputs.error && 'There were conflicts during the cherry-pick. These were commited without any resolving. Please resolve them manually and push the result to this branch before merging.' || 'The cherry-pick was successful without any conflicts. You should be able to simply merge this PR.' }}"
+        reviewers: ${{ github.event.commits[0].author.username }}
+        assignees: ${{ github.event.commits[0].author.username }}
+        labels: staging-port
+        milestone: 4

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "openslides-meta"]
+	path = global/meta
+	url = https://github.com/OpenSlides/openslides-meta.git

Dockerfile

@@ -15,10 +15,13 @@ EXPOSE 9002
 EXPOSE 9003
 ENV PYTHONPATH /app
 
-COPY scripts scripts
-COPY entrypoint.sh ./
-COPY openslides_backend openslides_backend
-COPY global global
+COPY --chown=appuser:appuser scripts scripts
+COPY --chown=appuser:appuser entrypoint.sh ./
+COPY --chown=appuser:appuser openslides_backend openslides_backend
+COPY --chown=appuser:appuser global global
+
+ARG VERSION=dev
+RUN echo "$VERSION" > openslides_backend/version.txt
 
 ENV EMAIL_HOST postfix
 ENV EMAIL_PORT 25

Makefile

@@ -1,19 +1,30 @@
 # Development and testing inside docker container or without docker (only unit and integration tests)
 
-paths = openslides_backend/ tests/ cli/
+paths = openslides_backend/ tests/ cli/ global/meta/dev/src/
 
-all: black autoflake isort flake8 mypy
+all: pyupgrade black autoflake isort flake8 mypy
+
+pyupgrade:
+	pyupgrade --py310-plus --exit-zero-even-if-changed $$(find . -name '*.py')
+
+check-pyupgrade:
+	pyupgrade --py310-plus $$(find . -name '*.py')
 
 black:
 	black $(paths)
 
+check-black:
+	black --check --diff $(paths)
+
 autoflake:
-	autoflake --verbose --in-place --remove-all-unused-imports \
-	--ignore-init-module-imports --recursive $(paths)
+	autoflake $(paths)
 
 isort:
 	isort $(paths)
 
+check-isort:
+	isort --check-only --diff $(paths)
+
 flake8:
 	flake8 $(paths)
 
@@ -29,14 +40,14 @@ test-unit-integration:
 check-all: validate-models-yml check-models check-initial-data-json check-example-data-json check-permissions
 
 validate-models-yml:
-	python cli/validate.py
+	make -C global/meta/dev validate-models
 
 generate-models:
 	python cli/generate_models.py $(MODELS_PATH)
 	black openslides_backend/models/models.py
 
 check-models:
-	python cli/generate_models.py check
+	python cli/generate_models.py --check
 
 generate-permissions:
 	python cli/generate_permissions.py $(MODELS_PATH)

README.md

@@ -13,6 +13,9 @@ We recently updated the source language in the database from english to the user
 
 ## Development
 
+Documentation for the actions: [Action overview](docs/Actions-Overview.md)
+Documentation for the presenters: [Presenter overview](docs/Presenter-Overview.md)
+
 ### Development with Docker Compose
 
 The setup is structured to do all development inside the docker containers. To start everything at once and get entered into a bash shell, run
@@ -185,9 +188,11 @@ General schema for internal routes: `/internal/<route>`
   and created write requests will have id -1.
 * `/system/action/health`: Return `{"status": "running"}` if successful. Useful for status checks against the backend.
 * `/system/action/info`: Returns a list of all possible actions with their respective JSON schema.
+* `/system/action/version`: Returns the current backend version.
 * `/internal/migrations`: Provides remote access to the migration tool. For more information, take a look at the [migration route docs](/docs/migration_route.md)
 
 ### Presenter Service
 
 * `/system/presenter/handle_request`: Main route of the service, is used to fetch presenter results.
 * `/system/presenter/health`: Return `{"status": "running"}` if successful. Useful for status checks against the backend.
+* `/system/presenter/version`: Returns the current backend version.