Issue 3720 pipectl: update install procedure to be able to work with …

…GitLab v17 (#3721)
epam · Sep 30, 2024 · c4edd93 · c4edd93
1 parent 0b22037
commit c4edd93
Show file tree

Hide file tree

Showing 6 changed files with 345 additions and 11 deletions.
diff --git a/deploy/contents/install/app/configure-utils.sh b/deploy/contents/install/app/configure-utils.sh
@@ -529,6 +529,9 @@ function api_register_gitlab {
     api_preference_append_array "$(api_preference_get_templated "git.token"          "$gitlab_root_token"                                        "false")"
     api_preference_append_array "$(api_preference_get_templated "git.user.id"        "1"                                                         "false")"
     api_preference_append_array "$(api_preference_get_templated "git.host"           "https://$CP_GITLAB_INTERNAL_HOST:$CP_GITLAB_INTERNAL_PORT" "true")"
+    if [ "$CP_GITLAB_VERSION" != "9" ]; then
+        api_preference_append_array "$(api_preference_get_templated "git.gitlab.api.version" "v4" "false")"
+    fi
     api_set_preference "$(api_preference_get_array)"
     api_preference_drop_array
 }

diff --git a/deploy/contents/install/app/install.sh b/deploy/contents/install/app/install.sh
@@ -801,6 +801,20 @@ fi
 if is_service_requested cp-gitlab-db; then
     print_ok "[Starting GitLab postgres DB deployment]"
 
+    if [ "$CP_GITLAB_VERSION" == "9" ]; then
+        if [ "$GITLAB_DATABASE_VERSION" != "9.6" ]; then
+            print_warn "CP_GITLAB_VERSION is 9 and GITLAB_DATABASE_VERSION is $GITLAB_DATABASE_VERSION, but probably should be 9.6! Installation will continue, but may fail."
+        fi
+    elif [ "$CP_GITLAB_VERSION" == "15" ]; then
+        if [ "$GITLAB_DATABASE_VERSION" != "12.4" ]; then
+            print_warn "CP_GITLAB_VERSION is 15 and GITLAB_DATABASE_VERSION is $GITLAB_DATABASE_VERSION, but probably should be 12.4! Installation will continue, but may fail."
+        fi
+    elif [ "$CP_GITLAB_VERSION" == "17" ]; then
+        if [ "$GITLAB_DATABASE_VERSION" != "14.11" ]; then
+            print_warn "CP_GITLAB_VERSION is 17 and GITLAB_DATABASE_VERSION is $GITLAB_DATABASE_VERSION, but probably should be 14.11! Installation will continue, but may fail."
+        fi
+    fi
+
     print_info "-> Deleting existing instance of GitLab postgres DB"
     delete_deployment_and_service   "cp-gitlab-db" \
                                     "/opt/gitlab-postgresql"
@@ -857,6 +871,20 @@ if is_service_requested cp-git; then
                                         $CP_GITLAB_INTERNAL_HOST
 
         print_info "-> Deploying GitLab"
+
+        if [ "$CP_GITLAB_VERSION" == "17" ]; then
+            export CP_GITLAB_SESSION_API_DISABLE="true"
+            if [ "$GITLAB_ROOT_PASSWORD" == "Passw0rd" ]; then
+                print_ok "CP_GITLAB_VERSION is 17 and GITLAB_ROOT_PASSWORD was not provided, will generate random password."
+                GITLAB_ROOT_PASSWORD=$(openssl rand -hex 8)
+                export GITLAB_ROOT_PASSWORD
+                update_config_value "$CP_INSTALL_CONFIG_FILE" \
+                                               "GITLAB_ROOT_PASSWORD" \
+                                               "$GITLAB_ROOT_PASSWORD"
+                init_kube_config_map
+            fi
+        fi
+
         set_kube_service_external_ip CP_GITLAB_SVC_EXTERNAL_IP_LIST \
                                      CP_GITLAB_NODE_IP \
                                      CP_GITLAB_KUBE_NODE_NAME \
@@ -892,9 +920,14 @@ if is_service_requested cp-git; then
             done
         else
             print_info "-> Setting GitLab root's private_token"
+            gitlab_token_expiration=""
+            if [ "$CP_GITLAB_VERSION" == "17" ]; then
+              gitlab_token_expiration=", expires_at: 365.days.from_now"
+            fi
+
             GITLAB_ROOT_TOKEN=$(openssl rand -hex 20)
-            gitlab_access_tokens_scopes=${CP_GITLAB_ACCESS_TOKEN_SCOPES:-":read_user,:read_repository,:api,:read_api,:write_repository,:sudo"}
-            gitlab_set_token_cmd="token=User.find_by_username('$GITLAB_ROOT_USER').personal_access_tokens.create(scopes:[$gitlab_access_tokens_scopes], name:'CloudPipelineRootToken'); token.set_token('$GITLAB_ROOT_TOKEN'); token.save!"
+            gitlab_access_tokens_scopes=${CP_GITLAB_ACCESS_TOKEN_SCOPES:-"'read_user','read_repository','api','read_api','write_repository','sudo'"}
+            gitlab_set_token_cmd="token=User.find_by_username('$GITLAB_ROOT_USER').personal_access_tokens.create(scopes:[$gitlab_access_tokens_scopes], name:'CloudPipelineRootToken'$gitlab_token_expiration); token.set_token('$GITLAB_ROOT_TOKEN'); token.save!"
             gitlab_set_token_response=$(execute_deployment_command cp-git cp-git "gitlab-rails runner \"$gitlab_set_token_cmd\"")
             if [ $? -ne 0 ]; then
                 print_err "Error occurred during adding GitLab root's private_token"
@@ -913,14 +946,35 @@ if is_service_requested cp-git; then
             init_kube_config_map
 
             print_info "Waiting $CP_GITLAB_INIT_TIMEOUT seconds, before getting impersonation token (while root token is retrieved - gitlab may still fail with 502)"
-            print_info "-> Getting GitLab root's impersonation token"
             sleep $CP_GITLAB_INIT_TIMEOUT
-            GITLAB_IMP_TOKEN=$(curl -k \
-                                    --request POST \
-                                    --silent \
-                                    --header "PRIVATE-TOKEN: $GITLAB_ROOT_TOKEN" \
-                                    --data "name=CloudPipeline" \
-                                    --data "scopes[]=api" https://$CP_GITLAB_INTERNAL_HOST:$CP_GITLAB_EXTERNAL_PORT/api/v4/users/1/impersonation_tokens | jq -r '.token')
+
+            # Enable web hooks to enable repository indexing for elastic search agent
+            if [ "$CP_GITLAB_VERSION" != "9" ]; then
+                print_info "-> Enabling allow_local_requests_from_web_hooks_and_services in GitLab settings..."
+                curl -k \
+                     --request PUT --header "PRIVATE-TOKEN: $GITLAB_ROOT_TOKEN" \
+                     "https://$CP_GITLAB_INTERNAL_HOST:$CP_GITLAB_EXTERNAL_PORT/api/v4/application/settings?allow_local_requests_from_web_hooks_and_services=true" &> /dev/null
+            fi
+
+            print_info "-> Getting GitLab root's impersonation token"
+            if [ "$CP_GITLAB_VERSION" == "17" ]; then
+                GITLAB_IMP_TOKEN=$(curl -k \
+                                      --request POST \
+                                      --silent \
+                                      --header "PRIVATE-TOKEN: $GITLAB_ROOT_TOKEN" \
+                                      --data "name=CloudPipeline" \
+                                      --data "expires_at=$(date +%Y-%m-%d -d'1 year')" \
+                                      --data "scopes[]=api" https://$CP_GITLAB_INTERNAL_HOST:$CP_GITLAB_EXTERNAL_PORT/api/v4/users/1/impersonation_tokens | jq -r '.token')
+
+            else
+                GITLAB_IMP_TOKEN=$(curl -k \
+                                      --request POST \
+                                      --silent \
+                                      --header "PRIVATE-TOKEN: $GITLAB_ROOT_TOKEN" \
+                                      --data "name=CloudPipeline" \
+                                      --data "scopes[]=api" https://$CP_GITLAB_INTERNAL_HOST:$CP_GITLAB_EXTERNAL_PORT/api/v4/users/1/impersonation_tokens | jq -r '.token')
+            fi
+
             if [ "$GITLAB_IMP_TOKEN" ] && [ "$GITLAB_IMP_TOKEN" != "null" ]; then
                 print_ok "GitLab impersonation token retrieved: $GITLAB_IMP_TOKEN"
                 export GITLAB_IMP_TOKEN

diff --git a/deploy/contents/install/install-config b/deploy/contents/install/install-config
@@ -60,7 +60,13 @@ CP_PREF_API_STATIC_DIRECTORY_EXT=/opt/api/ext/templates/static
 CP_PREF_API_ETC_DIRECTORY=/opt/api/etc
 
 # cp-git config
+
+# possible values 9, 15, 17
+CP_GITLAB_VERSION=9
+
+# possible values 12.4 for CP_GITLAB_VERSION=15, and 14.11 for CP_GITLAB_VERSION=17
 GITLAB_DATABASE_VERSION=9.6
+
 GITLAB_DATABASE_SHARED_BUFFERS=128MB
 GITLAB_DATABASE_MAX_CONNECTIONS=100
 GITLAB_DATABASE_ADAPTER=postgresql
@@ -77,7 +83,6 @@ CP_GITLAB_INTERNAL_HOST=cp-git.default.svc.cluster.local
 CP_GITLAB_INTERNAL_PORT=30080
 CP_GITLAB_EXTERNAL_PORT=30080
 CP_GITLAB_CERT_DIR=/opt/gitlab/pki
-CP_GITLAB_VERSION=9
 
 # cp-idp config
 IDP_HOST=cp-idp.default.svc.cluster.local

diff --git a/deploy/docker/build-dockers.sh b/deploy/docker/build-dockers.sh
@@ -184,14 +184,22 @@ docker build    $DOCKERS_SOURCES_PATH/cp-git \
                 -f $DOCKERS_SOURCES_PATH/cp-git/Dockerfile.9.4
 docker push "$CP_GITLAB_DIST_NAME"
 
-# 15.4.3 version
+# 15.5.4 version
 CP_GITLAB_15_DIST_NAME=${CP_GITLAB_15_DIST_NAME:-"$CP_DIST_REPO_NAME:git-15-${DOCKERS_VERSION}"}
 docker build    $DOCKERS_SOURCES_PATH/cp-git \
                 -t "$CP_GITLAB_15_DIST_NAME" \
                 -f $DOCKERS_SOURCES_PATH/cp-git/Dockerfile.15.5 \
                 --build-arg BASE_IMAGE="gitlab/gitlab-ce:15.5.4-ce.0"
 docker push "$CP_GITLAB_15_DIST_NAME"
 
+# 17.4.1 version
+CP_GITLAB_17_DIST_NAME=${CP_GITLAB_17_DIST_NAME:-"$CP_DIST_REPO_NAME:git-17-${DOCKERS_VERSION}"}
+docker build    $DOCKERS_SOURCES_PATH/cp-git \
+                -t "$CP_GITLAB_17_DIST_NAME" \
+                -f $DOCKERS_SOURCES_PATH/cp-git/Dockerfile.17.4 \
+                --build-arg BASE_IMAGE="gitlab/gitlab-ce:17.4.1-ce.0"
+docker push "$CP_GITLAB_17_DIST_NAME"
+
 # Notifier
 CP_NOTIFIER_DIST_NAME=${CP_NOTIFIER_DIST_NAME:-"$CP_DIST_REPO_NAME:notifier-${DOCKERS_VERSION}"}
 docker build    $DOCKERS_SOURCES_PATH/cp-notifier \

diff --git a/deploy/docker/cp-git/Dockerfile.17.4 b/deploy/docker/cp-git/Dockerfile.17.4
@@ -0,0 +1,32 @@
+# Copyright 2017-2024 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_IMAGE="gitlab/gitlab-ce:17.4.1-ce.0"
+
+FROM $BASE_IMAGE
+
+ADD update_config_v17.sh /update_config.sh
+RUN chmod +x /update_config.sh
+
+RUN curl -k -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh | bash && \
+    apt install -y gitlab-runner=17.4.0-1 sshpass
+
+ADD gitlab-runner-scripts /gitlab-runner-scripts
+RUN chmod +x /gitlab-runner-scripts/*.sh
+
+RUN curl -k -sL "https://cloud-pipeline-oss-builds.s3.amazonaws.com/tools/jq/jq-1.6/jq-linux64" -o /usr/bin/jq && \
+    chmod +x /usr/bin/jq
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD ["/update_config.sh; /assets/wrapper"]