Implement WebDAV server support in pipe

.github/workflows/main.yml

@@ -52,6 +52,24 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
       - run: bash deploy/github_actions/gha_build_pipe_linux_el6.sh
+  CLI_Linux_py39:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - run: bash deploy/github_actions/gha_build_pipe_linux_py39.sh
   CLI_MacOS:
     runs-on: macos-11
     steps:
@@ -221,7 +239,7 @@ jobs:
       - run: bash deploy/github_actions/gha_build_gpu_stat.sh
   Build_All:
     runs-on: ubuntu-20.04
-    needs: [CLI_Linux, CLI_Linux_el6, CLI_MacOS, CLI_Windows,
+    needs: [CLI_Linux, CLI_Linux_el6, CLI_Linux_py39, CLI_MacOS, CLI_Windows,
             Build_Client,
             Build_CloudData_Linux, Build_CloudData_Windows,
             Build_DataSharingService, Build_DataTransferService,

deploy/github_actions/gha_build_pipe_linux_py39.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2017-2024 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+CLOUD_PIPELINE_BUILD_NUMBER=$(($CLOUD_PIPELINE_BUILD_NUMBER_SEED+$GITHUB_RUN_NUMBER))
+
+./gradlew -PbuildNumber=${CLOUD_PIPELINE_BUILD_NUMBER}.${GITHUB_SHA} \
+          -Pprofile=release \
+          pipe-cli:buildLinuxPy39 \
+          --no-daemon \
+          -x :pipe-cli:test
+
+ls -lh pipe-cli/dist/dist-file/pipe \
+       pipe-cli/dist/dist-folder/pipe.tar.gz
+
+if [ "$GITHUB_REPOSITORY" == "epam/cloud-pipeline" ]; then
+    aws s3 cp --no-progress pipe-cli/dist/dist-file/pipe s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-py39
+    aws s3 cp --no-progress pipe-cli/dist/dist-folder/pipe.tar.gz s3://cloud-pipeline-oss-builds/temp/$CLOUD_PIPELINE_BUILD_NUMBER/pipe-py39.tar.gz
+fi

pipe-cli/build.gradle

@@ -50,6 +50,34 @@ task buildLinux(type: Exec, dependsOn: ':pipe-cli:test') {
     }
 }
 
+task buildLinuxPy39(type: Exec) {
+    group = 'build'
+    doFirst {
+        setCLIVersion()
+    }
+
+    environment "_BUILD_DOCKER_IMAGE": "quay.io/lifescience/cloud-pipeline:python3.9-centos7"
+    environment "_BUILD_SCRIPT_NAME": "$project.rootDir/pipe-cli/build_linux_py39.sh"
+
+    environment "PIPE_CLI_SOURCES_DIR": "$project.rootDir/pipe-cli"
+    environment "PIPE_CLI_LINUX_DIST_DIR": "$project.rootDir/pipe-cli"
+    environment "PIPE_MOUNT_SOURCES_DIR": "$project.rootDir/pipe-cli/mount"
+
+    commandLine "bash", "$project.rootDir/pipe-cli/build_docker.sh"
+    doLast {
+        cleanCLIVersion()
+        delete("$project.rootDir/api/src/main/resources/static/pipe")
+        copy {
+            from("$project.rootDir/pipe-cli/dist/dist-file/pipe")
+            into("$project.rootDir/api/src/main/resources/static/")
+        }
+        copy {
+            from("$project.rootDir/pipe-cli/dist/dist-folder/pipe.tar.gz")
+            into("$project.rootDir/api/src/main/resources/static/")
+        }
+    }
+}
+
 task buildWin(type: Exec, dependsOn: ':pipe-cli:test') {
     group = 'build'
     doFirst {

pipe-cli/build_docker.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright 2017-2020 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd $PIPE_CLI_SOURCES_DIR
+PIPE_COMMIT_HASH=$(git log --pretty=tformat:"%H" -n1 .)
+cd -
+
+docker pull $_BUILD_DOCKER_IMAGE &> /dev/null
+docker run -i --rm \
+           -v $PIPE_CLI_SOURCES_DIR:$PIPE_CLI_SOURCES_DIR \
+           -v $PIPE_CLI_LINUX_DIST_DIR:$PIPE_CLI_LINUX_DIST_DIR \
+           -v $_BUILD_SCRIPT_NAME:$_BUILD_SCRIPT_NAME \
+           --env PIPE_CLI_SOURCES_DIR=$PIPE_CLI_SOURCES_DIR \
+           --env PIPE_MOUNT_SOURCES_DIR=$PIPE_MOUNT_SOURCES_DIR \
+           --env PIPE_CLI_LINUX_DIST_DIR=$PIPE_CLI_LINUX_DIST_DIR \
+           --env PIPE_COMMIT_HASH=$PIPE_COMMIT_HASH \
+           $_BUILD_DOCKER_IMAGE \
+           bash $_BUILD_SCRIPT_NAME

pipe-cli/build_linux_py39.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+# Copyright 2017-2021 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+build_os_id=''
+build_os_version_id=''
+if [ -f "/etc/os-release" ]; then
+    source /etc/os-release
+    build_os_id="${ID}"
+    build_os_version_id="${VERSION_ID}"
+elif [ -f "/etc/centos-release" ]; then
+    build_os_id="centos"
+    build_os_version_id=$(cat /etc/centos-release | tr -dc '0-9.'|cut -d \. -f1)
+fi
+
+if [ -f /opt/miniconda/etc/profile.d/conda.sh ]; then
+    source /opt/miniconda/etc/profile.d/conda.sh
+    conda activate python
+fi
+
+###
+# Setup Pyinstaller
+###
+python -m pip install pyinstaller==5.13.2
+
+###
+# Setup common dependencies
+###
+python -m pip install macholib==1.16.2
+python -m pip install -r ${PIPE_CLI_SOURCES_DIR}/requirements.txt
+
+###
+# Build pipe fuse
+###
+cp ${PIPE_MOUNT_SOURCES_DIR}/libfuse/libfuse.so.2.9.2 ${PIPE_MOUNT_SOURCES_DIR}/libfuse/libfuse.so.frozen
+python -m pip install -r ${PIPE_MOUNT_SOURCES_DIR}/requirements.txt
+cd $PIPE_MOUNT_SOURCES_DIR && \
+pyinstaller \
+    --paths "$PIPE_CLI_SOURCES_DIR" \
+    --paths "$PIPE_MOUNT_SOURCES_DIR" \
+    --hidden-import=UserList \
+    --hidden-import=UserString \
+    --hidden-import=commands \
+    --hidden-import=ConfigParser \
+    --hidden-import=UserDict \
+    --hidden-import=itertools \
+    --hidden-import=collections \
+    --hidden-import=future.backports.misc \
+    --hidden-import=commands \
+    --hidden-import=base64 \
+    --hidden-import=__builtin__ \
+    --hidden-import=math \
+    --hidden-import=reprlib \
+    --hidden-import=functools \
+    --hidden-import=re \
+    --hidden-import=subprocess \
+    --hidden-import=_sysconfigdata \
+    --exclude-module=_tkinter \
+    --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \
+    --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \
+    -y \
+    --clean \
+    --distpath /tmp/mount/dist \
+    --add-data "${PIPE_MOUNT_SOURCES_DIR}/libfuse/libfuse.so.frozen:libfuse" \
+    ${PIPE_MOUNT_SOURCES_DIR}/pipe-fuse.py
+
+chmod +x /tmp/mount/dist/pipe-fuse/pipe-fuse
+
+###
+# Build pipe
+###
+function build_pipe {
+    local distpath="$1"
+    local onefile="$2"
+
+    version_file="${PIPE_CLI_SOURCES_DIR}/src/version.py"
+
+    bundle_type="one-folder"
+    [ "$onefile" ] && bundle_type="one-file"
+
+    sed -i '/__bundle_info__/d' $version_file
+    echo "__bundle_info__ = { 'bundle_type': '$bundle_type', 'build_os_id': '$build_os_id', 'build_os_version_id': '$build_os_version_id' }" >> $version_file
+
+    if [ "$PIPE_COMMIT_HASH" ]; then
+        sed -i '/__component_version__/d' $version_file
+        echo "__component_version__='$PIPE_COMMIT_HASH'" >> $version_file
+    fi
+
+    cd $PIPE_CLI_SOURCES_DIR && \
+    pyinstaller \
+        --paths "$PIPE_CLI_SOURCES_DIR" \
+        --hidden-import=UserList \
+        --hidden-import=UserString \
+        --hidden-import=commands \
+        --hidden-import=ConfigParser \
+        --hidden-import=UserDict \
+        --hidden-import=itertools \
+        --hidden-import=collections \
+        --hidden-import=future.backports.misc \
+        --hidden-import=commands \
+        --hidden-import=base64 \
+        --hidden-import=__builtin__ \
+        --hidden-import=math \
+        --hidden-import=reprlib \
+        --hidden-import=functools \
+        --hidden-import=re \
+        --hidden-import=subprocess \
+        --hidden-import=_sysconfigdata \
+        --exclude-module=_tkinter \
+        --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks" \
+        --additional-hooks-dir="${PIPE_MOUNT_SOURCES_DIR}/hooks-py39" \
+        -y \
+        --clean \
+        --distpath $distpath \
+        --add-data /tmp/mount/dist/pipe-fuse:mount \
+        --add-data "$PIPE_CLI_SOURCES_DIR/res/effective_tld_names.dat.txt:tld/res/" \
+        ${PIPE_CLI_SOURCES_DIR}/pipe.py $onefile
+}
+
+build_pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file --onefile
+mv $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file/pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-file/pipe
+
+build_pipe $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder
+tar -zcf $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder/pipe.tar.gz \
+        -C $PIPE_CLI_LINUX_DIST_DIR/dist/dist-folder \
+        pipe

pipe-cli/docker/python3.9-centos7/Dockerfile

@@ -0,0 +1,23 @@
+# Copyright 2017-2024 EPAM Systems, Inc. (https://www.epam.com/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM centos:7
+
+RUN yum install -y gcc \
+                   wget
+
+RUN wget "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O /tmp/miniconda.sh && \
+    bash /tmp/miniconda.sh -b -p /opt/miniconda && \
+    source /opt/miniconda/etc/profile.d/conda.sh && \
+    conda create -y -n python python=3.9