Merge branch 'HDDS-10239-container-reconciliation' into HDDS-11290-sc…

…an-on-error

* HDDS-10239-container-reconciliation: (428 commits)
  HDDS-11081. Use thread-local instance of FileSystem in Freon tests (apache#7091)
  HDDS-11333. Avoid hard-coded current version in upgrade/xcompat tests (apache#7089)
  Mark TestPipelineManagerMXBean#testPipelineInfo as flaky
  Mark TestAddRemoveOzoneManager#testForceBootstrap as flaky
  HDDS-11352. HDDS-11353. Mark TestOzoneManagerHAWithStoppedNodes as flaky
  HDDS-11354. Mark TestOzoneManagerSnapshotAcl#testLookupKeyWithNotAllowedUserForPrefixAcl as flaky
  HDDS-11355. Mark TestMultiBlockWritesWithDnFailures#testMultiBlockWritesWithIntermittentDnFailures as flaky
  HDDS-11227. Use server default key provider to encrypt/decrypt keys from multiple OMs. (apache#7081)
  HDDS-11316. Improve Create Key and Chunk IO Dashboards (apache#7075)
  HDDS-11239. Fix KeyOutputStream's exception handling when calling hsync concurrently (apache#7047)
  HDDS-11254. Reconcile commands should be handled by datanode ReplicationSupervisor (apache#7076)
  HDDS-11331. Fix Datanode unable to report for a long time (apache#7090)
  HDDS-11346. FS CLI gives incorrect recursive volume deletion prompt (apache#7102)
  HDDS-11349. Add NullPointer handling when volume/bucket tables are not initialized (apache#7103)
  HDDS-11209. Avoid insufficient EC pipelines in the container pipeline cache (apache#6974)
  HDDS-11284. refactor quota repair non-blocking while upgrade (apache#7035)
  HDDS-9790. Add tests for Overview page (apache#6983)
  HDDS-10904. [hsync] Enable PutBlock piggybacking and incremental chunk list by default (apache#7074)
  HDDS-11322. [hsync] Block ECKeyOutputStream from calling hsync and hflush (apache#7098)
  HDDS-11325. Intermittent failure in TestBlockOutputStreamWithFailures#testContainerClose (apache#7099)
  ...

Conflicts:
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java

.github/workflows/build-ratis.yml

@@ -84,15 +84,15 @@ jobs:
       - name: Get component versions
         id: versions
         run: |
-          thirdparty_version="$(mvn help:evaluate -N -q -DforceStdout -Dexpression=ratis.thirdparty.version)"
+          thirdparty_version="$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=ratis.thirdparty.version)"
           echo "thirdparty=${thirdparty_version}" >> $GITHUB_OUTPUT
 
           ratis_sha=$(git rev-parse --short HEAD)
-          ratis_version="$(mvn help:evaluate -N -q -DforceStdout -Dexpression=project.version | sed -e "s/-SNAPSHOT/-${ratis_sha}-SNAPSHOT/")"
+          ratis_version="$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=project.version | sed -e "s/-SNAPSHOT/-${ratis_sha}-SNAPSHOT/")"
           echo "ratis=${ratis_version}" >> $GITHUB_OUTPUT
       - name: Run a full build
         run: |
-          mvn versions:set -DnewVersion=${{ steps.versions.outputs.ratis }}
+          mvn -B --no-transfer-progress -Dscan=false versions:set -DnewVersion=${{ steps.versions.outputs.ratis }}
           dev-support/checks/build.sh
       - name: Store Maven repo for tests
         uses: actions/upload-artifact@v4
@@ -119,9 +119,9 @@ jobs:
       - name: Get component versions
         id: versions
         run: |
-          echo "grpc=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.grpc.version)" >> $GITHUB_OUTPUT
-          echo "netty=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.netty.version)" >> $GITHUB_OUTPUT
-          echo "protobuf=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.protobuf.version)" >> $GITHUB_OUTPUT
+          echo "grpc=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.grpc.version)" >> $GITHUB_OUTPUT
+          echo "netty=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.netty.version)" >> $GITHUB_OUTPUT
+          echo "protobuf=$(mvn help:evaluate -N -q -DforceStdout -Dscan=false -Dexpression=shaded.protobuf.version)" >> $GITHUB_OUTPUT
   debug:
     runs-on: ubuntu-20.04
     needs:

.github/workflows/ci-with-ratis.yml

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow tests Ozone with custom build of Ratis.
+#
+# Inputs:
+# - Ozone commit to build
+# - Ratis repo
+# - Ratis commit to build
+
+name: ci-with-ratis
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: Ozone ref (branch, tag or commit SHA)
+        default: master
+        required: true
+      ratis-repo:
+        description: Ratis repository
+        default: 'apache/ratis'
+        required: true
+      ratis-ref:
+        description: Ratis ref (branch, tag or commit SHA)
+        default: 'master'
+        required: true
+run-name: Test Ozone ${{ inputs.ref }} with Ratis ${{ inputs.ratis-repo }} @ ${{ inputs.ratis-ref }}
+jobs:
+  ratis:
+    uses: ./.github/workflows/build-ratis.yml
+    with:
+      repo: ${{ github.event.inputs.ratis-repo || format('{0}/ratis', github.repository_owner) }}
+      ref: ${{ github.event.inputs.ratis-ref }}
+  ozone:
+    needs:
+      - ratis
+    uses: ./.github/workflows/ci.yml
+    secrets: inherit
+    with:
+      ratis_args: "-Dratis.version=${{ needs.ratis.outputs.ratis-version }} -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }} -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version }} -Dnetty.version=${{ needs.ratis.outputs.netty-version }} -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}"
+      ref: ${{ github.event.inputs.ref }}