GUI redundant requests fixes (#3347) Library

client/src/components/pipelines/browser/Metadata.js

@@ -194,8 +194,6 @@ function makeCurrentOrderSort (array) {
     folder: componentParameters.id ? folders.load(componentParameters.id) : pipelinesLibrary,
     folderId: componentParameters.id,
     metadataClass: componentParameters.class,
-    entityFields: new MetadataEntityFields(componentParameters.id),
-    metadataClasses: new MetadataClassLoadAll(),
     onReloadTree: params.onReloadTree,
     authenticatedUserInfo,
     preferences,
@@ -218,6 +216,10 @@ export default class Metadata extends React.Component {
   };
 
   @observable keys;
+  @observable metadataFieldsRequest;
+  @observable metadataFieldsPending = false;
+  @observable metadataClassesRequest;
+  @observable metadataClassesPending = false;
 
   metadataRequest = {};
   externalMetadataEntity = {};
@@ -264,14 +266,28 @@ export default class Metadata extends React.Component {
 
   uploadButton;
 
+  @computed
+  get entityFields () {
+    if (this.metadataFieldsRequest && this.metadataFieldsRequest.loaded) {
+      return this.metadataFieldsRequest.value || [];
+    }
+    return null;
+  }
+
+  @computed
+  get metadataClasses () {
+    if (this.metadataClassesRequest && this.metadataClassesRequest.loaded) {
+      return this.metadataClassesRequest.value || [];
+    }
+    return null;
+  }
+
   @computed
   get entityTypes () {
-    const {entityFields, metadataClasses} = this.props;
-    if (entityFields.loaded && metadataClasses.loaded) {
-      const mappedEntityFields = (entityFields.value || [])
-        .map(e => e);
+    if (this.entityFields && this.metadataClasses) {
+      const mappedEntityFields = this.entityFields.map(e => e);
       const ignoreClasses = new Set(mappedEntityFields.map(f => f.metadataClass.id));
-      const otherClasses = (metadataClasses.value || [])
+      const otherClasses = this.metadataClasses
         .filter(({id}) => !ignoreClasses.has(id))
         .map(metadataClass => ({
           fields: [],
@@ -470,7 +486,7 @@ export default class Metadata extends React.Component {
         message.error(request.error, 5);
       } else {
         this.clearSelection();
-        await this.props.entityFields.fetch();
+        await this.loadMetadataFields(this.props.folderId);
         await this.loadColumns({append: true});
         await this.loadData();
         await this.props.folder.fetch();
@@ -1870,7 +1886,7 @@ export default class Metadata extends React.Component {
             parentId={currentItem ? currentItem.parent.id : null}
             currentItem={this.state.selectedItem}
             onUpdateMetadata={async () => {
-              await this.props.entityFields.fetch();
+              await this.loadMetadataFields(this.props.folderId);
               await this.loadColumns({append: true});
               await this.loadData();
               const [selectedItem] =
@@ -2659,7 +2675,7 @@ export default class Metadata extends React.Component {
             multiple={false}
             synchronous
             onRefresh={async () => {
-              await this.props.entityFields.fetch();
+              await this.loadMetadataFields(this.props.folderId);
               await this.props.folder.fetch();
               if (this.props.onReloadTree) {
                 this.props.onReloadTree(true);
@@ -2751,11 +2767,17 @@ export default class Metadata extends React.Component {
     authenticatedUserInfo
       .fetchIfNeededOrWait()
       .then(() => this.fetchDefaultMetadataProperties());
+    this.loadMetadataFields(this.props.folderId);
+    this.loadMetadataClasses();
     document.addEventListener('keydown', this.resetSelection);
     window.addEventListener('mouseup', this.handleFinishSelection);
   };
 
   componentDidUpdate (prevProps, prevState, snapshot) {
+    if (prevProps.folderId !== this.props.folderId) {
+      this.loadMetadataFields(this.props.folderId);
+      this.loadMetadataClasses();
+    }
     const metadataClassChanged = prevProps.metadataClass !== this.props.metadataClass;
     const folderChanged = prevProps.folderId !== this.props.folderId;
     const filtersChanged = metadataFilterUtilities
@@ -2821,7 +2843,9 @@ export default class Metadata extends React.Component {
         folderChanged
           ? this.fetchDefaultMetadataProperties(folderChanged)
           : this.loadColumns({reset: true}),
-        folderChanged ? this.props.entityFields.fetch() : Promise.resolve()
+        folderChanged
+          ? this.loadMetadataFields(this.props.folderId)
+          : Promise.resolve()
       ];
       return Promise.all(promises).then(() => {
         const {defaultOrderBy, columns} = this.state;
@@ -2840,6 +2864,35 @@ export default class Metadata extends React.Component {
     });
   };
 
+  loadMetadataFields = async (folderId) => {
+    if (
+      !folderId ||
+      (this.metadataFieldsRequest && this.metadataFieldsRequest.pending)
+    ) {
+      return;
+    }
+    this.metadataFieldsRequest = new MetadataEntityFields(folderId);
+    this.metadataFieldsPending = true;
+    await this.metadataFieldsRequest.fetch();
+    if (this.metadataFieldsRequest.error) {
+      message.error(this.metadataFieldsRequest.error, 5);
+    }
+    this.metadataFieldsPending = false;
+  };
+
+  loadMetadataClasses = async () => {
+    if (this.metadataClassesRequest && this.metadataClassesRequest.pending) {
+      return;
+    }
+    this.metadataClassesRequest = new MetadataClassLoadAll();
+    this.metadataClassesPending = true;
+    await this.metadataClassesRequest.fetch();
+    if (this.metadataClassesRequest.error) {
+      message.error(this.metadataClassesRequest.error, 5);
+    }
+    this.metadataClassesPending = false;
+  };
+
   fetchDefaultMetadataProperties = (folderChanged = false) => {
     const {
       defaultMetadataPropertiesFetched,

client/src/components/pipelines/browser/data-storage/index.js

@@ -137,12 +137,6 @@ const isStandardClass = (storageClass) =>
     path: decodeURIComponent(queryParameters.path || ''),
     showVersions,
     showArchives,
-    restoreInfo: new LifeCycleEffectiveHierarchy(
-      params.id,
-      decodeURIComponent(queryParameters.path || ''),
-      'FOLDER',
-      false
-    ),
     openPreview
   };
 })
@@ -177,6 +171,7 @@ export default class DataStorage extends React.Component {
   });
 
   @observable generateDownloadUrls;
+  @observable restoreInfoRequest;
 
   get showMetadata () {
     if (this.state.metadata === undefined && this.storage.info) {
@@ -344,11 +339,10 @@ export default class DataStorage extends React.Component {
   @computed
   get lifeCycleRestoreInfo () {
     const {
-      restoreInfo,
       path
     } = this.props;
-    if (restoreInfo && restoreInfo.loaded) {
-      const [first, ...rest] = restoreInfo.value || [];
+    if (this.restoreInfoRequest && this.restoreInfoRequest.loaded) {
+      const [first, ...rest] = this.restoreInfoRequest.value || [];
       const currentPath = path
         ? [
           !path.startsWith('/') && '/',
@@ -366,7 +360,7 @@ export default class DataStorage extends React.Component {
         parentRestore = first;
         currentRestores = rest;
       } else {
-        currentRestores = restoreInfo.value;
+        currentRestores = this.restoreInfoRequest.value;
       }
       return {
         parentRestore,
@@ -417,6 +411,24 @@ export default class DataStorage extends React.Component {
       : this.storage.writeAllowed;
   }
 
+  loadRestoreInfo = async (storageId, path = '') => {
+    if (!storageId) {
+      return;
+    }
+    this.restoreInfoRequest = new LifeCycleEffectiveHierarchy(
+      storageId,
+      decodeURIComponent(path),
+      'FOLDER',
+      false
+    );
+    this.restoreInfoPending = true;
+    await this.restoreInfoRequest.fetch();
+    if (this.restoreInfoRequest.error) {
+      message.error(this.restoreInfoRequest.error, 5);
+    }
+    this.restoreInfoPending = false;
+  };
+
   onDataStorageEdit = async (storage) => {
     if (!this.storage.info) {
       return;
@@ -808,15 +820,15 @@ export default class DataStorage extends React.Component {
   };
 
   restoreFiles = (payload) => {
-    const {storageId, restoreInfo} = this.props;
+    const {storageId} = this.props;
     const request = new LifeCycleRestoreCreate(storageId);
     this.setState({restorePending: true}, async () => {
       await request.send(payload);
       if (request.error) {
         message.error(request.error, 5);
         return this.setState({restorePending: false});
       }
-      restoreInfo.fetch()
+      this.loadRestoreInfo(this.props.storageId, this.props.path)
         .then(() => {
           this.setState({restorePending: false});
           this.closeRestoreFilesDialog();
@@ -2822,10 +2834,17 @@ export default class DataStorage extends React.Component {
       };
       this.openPreviewModal(file);
     }
+    this.loadRestoreInfo(this.props.storageId, this.props.path);
     this.updateStorageIfRequired();
   }
 
   componentDidUpdate (prevProps) {
+    if (
+      this.props.storageId !== prevProps.storageId ||
+      this.props.path !== prevProps.path
+    ) {
+      this.loadRestoreInfo(this.props.storageId, this.props.path);
+    }
     this.clearSelectedItemsIfRequired();
     this.updateStorageIfRequired();
   }

deploy/docker/cp-tools/base/rockylinux/vanilla/Dockerfile

@@ -26,8 +26,14 @@ RUN yum install -y wget \
                    git \
                    curl \
                    procps \
+                   findutils \
                    epel-release && \
     yum clean all && \
     yum install -y python2 && \
     ln -sf python2 /usr/bin/python && \
     curl https://cloud-pipeline-oss-builds.s3.amazonaws.com/tools/pip/2.7/get-pip.py | python -
+
+# Enable epel-release and powertools by default
+RUN yum install -y epel-release && \
+    yum config-manager --set-enabled powertools && \
+    yum update -y

scripts/pipeline-launch/launch.sh

@@ -434,6 +434,7 @@ function run_pre_common_commands {
 function define_distro_name_and_version {
       # Get the distro name and version
       CP_OS=
+      CP_OS_FAMILY=
       CP_VER=
       if [ -f /etc/os-release ]; then
             # freedesktop.org and systemd
@@ -458,8 +459,25 @@ function define_distro_name_and_version {
             CP_OS=$(uname -s)
             CP_VER=$(uname -r)
       fi
+
+      case $CP_OS in
+          ubuntu | debian)
+            CP_OS_FAMILY=debian
+            ;;
+          centos | rocky | fedora | ol | amzn)
+            CP_OS_FAMILY=rhel
+            ;;
+          *)
+            CP_OS_FAMILY=linux
+            ;;
+      esac
+
       export CP_OS
+      export CP_OS_FAMILY
       export CP_VER
+      export CP_VER_MAJOR="${CP_VER%%.*}"
+
+
 }
 
 # This function handle any distro/version - specific package manager state, e.g. clean up or reconfigure
@@ -480,7 +498,7 @@ function configure_package_manager {
             # System package manager setup
             local CP_REPO_BASE_URL_DEFAULT="${CP_REPO_BASE_URL_DEFAULT:-"${GLOBAL_DISTRIBUTION_URL}tools/repos"}"
             local CP_REPO_BASE_URL="${CP_REPO_BASE_URL_DEFAULT}/${CP_OS}/${CP_VER}"
-            if [ "$CP_OS" == "centos" ]; then
+            if [ "$CP_OS" == "centos" ] || [ "$CP_OS" == "rocky" ]; then
                   for _CP_REPO_RETRY_ITER in $(seq 1 $CP_REPO_RETRY_COUNT); do
                         # Remove nvidia repositories, as they cause run initialization failure
                         rm -f /etc/yum.repos.d/cuda.repo
@@ -548,8 +566,8 @@ function get_install_command_by_current_distr {
             _TOOLS_TO_INSTALL="$(sed "s/\( \|^\)ltdl\( \|$\)/ ${_ltdl_lib_name} /g" <<< "$_TOOLS_TO_INSTALL")"
       fi
       if [[ "$_TOOLS_TO_INSTALL" == *"python"* ]] && \
-         [ "$CP_OS" == "centos" ] && \
-         [ "$CP_VER" == "8" ]; then
+         { [ "$CP_OS" == "centos" ] || [ "$CP_OS" == "rocky" ]; } && \
+         { [[ "$CP_VER" == "8"* ]] || [[ "$CP_VER" == "9"* ]]; }; then
             _TOOLS_TO_INSTALL="$(sed -e "s/python/python2/g" <<< "$_TOOLS_TO_INSTALL")"
       fi
 
@@ -2151,7 +2169,7 @@ echo "-"
 # Force SystemD capability if the Kubernetes is requested
 if ( check_cp_cap "CP_CAP_SYSTEMD_CONTAINER" || check_cp_cap "CP_CAP_KUBE" ) \
     && check_installed "systemctl" && \
-    [ "$CP_OS" == "centos" ]; then
+    [ "$CP_OS" == "centos" ] || [ "$CP_OS" == "rocky" ]; then
         _SYSTEMCTL_STATUS=$(systemctl &> /dev/null; $?)
         if [ "$_SYSTEMCTL_STATUS" -eq 0 ]; then
             echo "Systemd already active, skipping installation"

workflows/pipe-common/shell/dind_setup

@@ -29,15 +29,12 @@ IS_RPM_BASED=$?
 ######################################################
 _DIND_DEP_INSTALL_RESULT=0
 if [ $IS_RPM_BASED -eq 0 ]; then
-    yum install -y -q \
-                btrfs-progs \
-                e2fsprogs \
-                iptables \
-                iproute \
-                xfsprogs \
-                xz \
-                pigz \
-                kmod
+    _PACKAGES_TO_INSTALL="e2fsprogs iptables iproute xfsprogs xz pigz kmod"
+    # Rocky doesn't have btrfs-progs
+    if [ "$CP_OS" != "rocky" ]; then
+        _PACKAGES_TO_INSTALL="${_PACKAGES_TO_INSTALL} btrfs-progs"
+    fi
+    yum install -y -q "$_PACKAGES_TO_INSTALL"
     _DIND_DEP_INSTALL_RESULT=$?
 else
     apt install -y -qq \
@@ -64,7 +61,13 @@ pipe_log_info "DIND dependencies installed" "$DIND_SETUP_TASK"
 ######################################################
 
 [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf
-export CP_CAP_DIND_VERSION="${CP_CAP_DIND_VERSION:-18.09.6}"
+
+CP_CAP_DIND_DEFAULT_VERSION=18.09.6
+if [ "$CP_OS" == "rocky" ]; then
+    CP_CAP_DIND_DEFAULT_VERSION=20.10.23
+fi
+
+export CP_CAP_DIND_VERSION="${CP_CAP_DIND_VERSION:-$CP_CAP_DIND_DEFAULT_VERSION}"
 export CP_CAP_DIND_CHANNEL="${CP_CAP_DIND_CHANNEL:-stable}"
 export CP_CAP_DIND_ARCH="${CP_CAP_DIND_ARCH:-x86_64}"
 
@@ -80,11 +83,20 @@ rm -f docker.tgz
 
 pipe_log_info "Docker installed: ${CP_CAP_DIND_CHANNEL}/${CP_CAP_DIND_ARCH}/docker-${CP_CAP_DIND_VERSION}" "$DIND_SETUP_TASK"
 
+
+# TODO fix for Rocky Linux
 nvidia-smi
 REQUIRES_GPU_DIND=$?
 if [ $REQUIRES_GPU_DIND -eq 0 ]; then
     pipe_log_info "Active CUDA environment detected, try to install NVIDIA Docker" "$DIND_SETUP_TASK"
-    distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+    distribution="${CP_OS}${CP_VER}"
+    if [ "$CP_OS" != "rocky" ]; then
+      if [[ "$CP_VER" == "8"* ]]; then
+        distribution="rhel8.6"
+      elif [[ "$CP_VER" == "9"* ]]; then
+        distribution="rhel9.0"
+      fi
+    fi
     if [ $IS_RPM_BASED -eq 0 ]; then
         export CP_CAP_DIND_GPU_VERSION="${CP_CAP_DIND_GPU_VERSION:-2.0.0-3.docker18.09.6}"
 
@@ -95,9 +107,6 @@ if [ $REQUIRES_GPU_DIND -eq 0 ]; then
         find /etc/yum.repos.d -type f \( -name "*nvidia*" -o -name "*docker*" \)  -exec rm -f {} \;
     else
         export CP_CAP_DIND_GPU_VERSION="${CP_CAP_DIND_GPU_VERSION:-2.0.0+docker18.09.6-3}"
-        os=$(. /etc/os-release;echo $ID)
-        os_release=$(. /etc/os-release;echo $VERSION_CODENAME)
-        sources_list=/etc/apt/sources.list
         curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add -
         curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list \
         | tee /etc/apt/sources.list.d/nvidia-docker.list