Initialize certs e2e tests 4

.dockerignore

@@ -25,7 +25,7 @@ grr/gui/static/bower_components
 grr/gui/static/node_modules
 grr/gui/static/tmp
 grr/var
-grr-server*.tar.gz
+grr_server*.tar.gz
 LICENSE
 README.md
 travis

.github/workflows/build.yml

@@ -248,17 +248,19 @@ jobs:
       - name: Start docker compose stack
         shell: bash
         run: |
+          docker_config_files/init_certs.sh
           docker compose \
-            -f docker-compose.yaml \
-            -f docker-compose.testing.yaml \
+            -f compose.yaml \
+            -f compose.testing.yaml \
             up -d --wait
       - name: Test
         shell: bash
         run: |
-          docker compose exec grr-client bash /configs/create_fake_user.sh
+          docker compose exec grr-client bash /configs/client/create_fake_user.sh
           docker run \
             --add-host=host.docker.internal:host-gateway \
             -v $(pwd):/github_workspace \
+            -v ./docker_config_files:/configs \
             -w /github_workspace \
             --entrypoint appveyor/e2e_tests/run_docker_compose_e2e_test.sh \
             ${{ env.DOCKER_REPOSITORY }}:testing \

CHANGELOG.md

@@ -14,8 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Removed support for foreman rules using `uname` of an endpoint (this can be
   simulated by using 3 rules for system name, release and version).
 * GRR server Debian package is removed when github actions are updated. The
-  docker image and docker compose stack (see section "Added") are the
+  docker image and Docker Compose stack (see section "Added") are the
   recommended wait of running GRR in the future.
+* Removed the `provides` field from the `Artifact` message. This change has been
+  done in anticipation of the removal of the same field from the official GitHub
+  repository (ForensicArtifacts/artifacts#275).
 
 
 ### Added

appveyor/e2e_tests/run_docker_compose_e2e_test.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 #
-# Runs the e2e tests in the docker-compose stack.
+# Runs the e2e tests in the Docker Compose stack.
 #
-# This script is executed in the grr docker container or in an
+# This script is executed in the grr Docker container or in an
 # environment with the grr src and develpment environment
 # (grr-python-api, grr-test) available. And assumes the
-# docker-compose stack to be running with exposed ports for
+# Docker Compose stack to be running with exposed ports for
 # the admin API and GRR database.
 #
 # Running this test (from the main folder):
-# - Start the docker compose stack with:
-#     $ docker-compose up
+# - Start the Docker Compose stack with:
+#     $ docker compose up
 #
-# - Build and run the GRR docker container and set the entrypoint
+# - Build and run the GRR Docker container and set the entrypoint
 #   to this script:
 #     $ docker build -f ./Dockerfile . -t local-grr-container
 #     $ docker run \
@@ -25,7 +25,7 @@
 
 set -ex
 
-# The IP address of the client inside the docker-compose stack.
+# The IP address of the client inside the Docker Compose stack.
 readonly CLIENT_IP=${1}
 
 readonly GRR_API="http://host.docker.internal:8000"

build_requirements.txt

@@ -0,0 +1,5 @@
+pip==24.0
+pytest==6.2.5
+pytest-xdist==2.2.1
+setuptools==69.5.1
+wheel==0.43.0

colab/grr_colab/fs.py

@@ -13,7 +13,6 @@
 from grr_colab import vfs
 from grr_response_proto import flows_pb2
 from grr_response_proto import jobs_pb2
-from grr_response_server.flows.general import file_finder
 
 
 class FileSystem(object):
@@ -208,9 +207,7 @@ def _collect_file(self, path: Text) -> None:
     args.action.action_type = flows_pb2.FileFinderAction.Action.DOWNLOAD
 
     try:
-      cff = self._client.CreateFlow(
-          name=file_finder.ClientFileFinder.__name__, args=args
-      )
+      cff = self._client.CreateFlow(name='FileFinder', args=args)
     except api_errors.AccessForbiddenError as e:
       raise errors.ApprovalMissingError(self.id, e)
 

colab/grr_colab/vfs.py

@@ -128,7 +128,7 @@ def detach(self) -> None:  # pytype: disable=signature-mismatch  # overriding-re
   def readable(self) -> bool:
     return True
 
-  def read(self, size: int = -1) -> bytes:
+  def read(self, size: int = -1) -> bytes:  # pytype: disable=signature-mismatch
     self._ensure_not_closed()
     size = size or -1
 

docker-compose.yaml → compose.yaml

@@ -1,3 +1,4 @@
+version: "3.8"
 services:
   db:
     image: mysql:8.2
@@ -26,37 +27,50 @@ services:
       retries: 10
 
   grr-admin-ui:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-admin-ui
     hostname: admin-ui
-    restart: always
     depends_on:
       db:
         condition: service_healthy
+      fleetspeak-admin:
+        condition: service_started
     volumes:
-      - ./docker_config_files/server:/configs/
+      - ./docker_config_files:/configs/
+      # Mount a directory for the repacked client installers, so they
+      # can be used in the grr-client container which mounts the same volume.
+      - client_installers:/client_installers
     ports:
       - "8000:8000"
     expose:
       - "8000"
     networks:
       - server-network
-    command:
-      - -component
-      - admin_ui
-      - -config
-      - /configs/grr.server.yaml
-      - --verbose
+    entrypoint: [
+      "/bin/bash",
+      "-c",
+      "/configs/server/repack_clients.sh && grr_server -component admin_ui -config /configs/server/grr.server.yaml --verbose"
+    ]
+    healthcheck:
+      # As soon as any files have been written to the /client_installer we
+      # assume the service is healthy.
+      test: |
+        if [[ -z "$(ls /client_installers)" ]]; then
+          echo "Healthckeck: GRR client installer not available"
+          exit 1
+        fi
+      timeout: 10s
+      retries: 10
 
   grr-fleetspeak-frontend:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-fleetspeak-frontend
     hostname: grr-fleetspeak-frontend
     depends_on:
       db:
         condition: service_healthy
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     expose:
       - "11111"
     restart: always
@@ -66,11 +80,11 @@ services:
       - -component
       - frontend
       - -config
-      - /configs/grr.server.yaml
+      - /configs/server/grr.server.yaml
       - --verbose
 
   fleetspeak-admin:
-    image: ghcr.io/google/fleetspeak:cl-601031487
+    image: ghcr.io/google/fleetspeak:latest
     container_name: fleetspeak-admin
     hostname: fleetspeak-admin
     depends_on:
@@ -81,20 +95,20 @@ services:
     expose:
       - "4444"
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     entrypoint: [
       "server",
       "-components_config",
-      "/configs/textservices/admin.components.config",
+      "/configs/server/textservices/admin.components.config",
       "-services_config",
-      "/configs/grr_frontend.service",
+      "/configs/server/grr_frontend.service",
       "-alsologtostderr",
       "-v",
       "1000"
     ]
 
   fleetspeak-frontend:
-    image: ghcr.io/google/fleetspeak:cl-601031487
+    image: ghcr.io/google/fleetspeak:latest
     container_name: fleetspeak-frontend
     hostname: fleetspeak-frontend
     depends_on:
@@ -106,23 +120,23 @@ services:
       - "4443"
       - "10000"
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     entrypoint: [
       "server",
       "-components_config",
-      "/configs/textservices/frontend.components.config",
+      "/configs/server/textservices/frontend.components.config",
       "-services_config",
-      "/configs/grr_frontend.service",
+      "/configs/server/grr_frontend.service",
       "-alsologtostderr",
       "-v",
       "1000"
     ]
 
   grr-worker:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ghcr.io/google/grr:latest
     container_name: grr-worker
     volumes:
-      - ./docker_config_files/server/:/configs/
+      - ./docker_config_files:/configs
     hostname: grr-worker
     depends_on:
       db:
@@ -134,27 +148,34 @@ services:
       - -component
       - worker
       - -config
-      - /configs/grr.server.yaml
+      - /configs/server/grr.server.yaml
       - --verbose
 
   grr-client:
-    image: ghcr.io/google/grr:docker-compose-testing
+    image: ubuntu:22.04
     container_name: grr-client
-    restart: always
     depends_on:
-      - db
-      - fleetspeak-frontend
+      db:
+        condition: service_healthy
+      fleetspeak-frontend:
+        condition: service_started
+      grr-admin-ui:
+        # Service is healthy as soon as client installers are repacked.
+        condition: service_healthy
     volumes:
-      - ./docker_config_files/client/:/configs/
-      # Mount the client_installers folder, to preserve
-      # the repacked templates across restarts.
+      - ./docker_config_files:/configs
+      # Mount the client_installers folder which contains the
+      # repacked templates written by the grr-admin-ui container
       - client_installers:/client_installers
+      # Mount the client_state volume to preserve the clients state
+      # including the client_id across restarts.
+      - client_state:/client_state
     networks:
       - server-network
     entrypoint: [
       "/bin/bash",
       "-c",
-      "/configs/repack_install_client.sh && fleetspeak-client -config /configs/client.config"
+      "/configs/client/install_client.sh && fleetspeak-client -config /configs/client/client.config"
     ]
     healthcheck:
       test: |
@@ -168,5 +189,6 @@ services:
 volumes:
   db_data:
   client_installers:
+  client_state:
 networks:
   server-network: