diff --git a/.gitlab-ci.yaml b/.gitlab-ci.yaml index d591e111be..0e37825122 100644 --- a/.gitlab-ci.yaml +++ b/.gitlab-ci.yaml @@ -665,17 +665,20 @@ broadcast_tests: test_tools_tests: stage: test extends: .test_tools_based - + needs: + - !reference [.test_tools_based, needs] + - job: copy_testing_block_logs_to_cache variables: PYTEST_TIMEOUT_MINUTES: 30 PYTEST_JUNIT_REPORT: $CI_PROJECT_DIR/tests/python/hive-local-tools/test-tools/tests/report.xml - + TESTING_BLOCK_LOGS_DESTINATION: $CI_PROJECT_DIR/tests/python/hive-local-tools/test-tools/tests/testing_block_logs script: - - cd tests/python/hive-local-tools/test-tools/tests + - cp -r $TESTING_BLOCK_LOGS_CACHE_DIR/* $TESTING_BLOCK_LOGS_DESTINATION + - cd $CI_PROJECT_DIR/tests/python/hive-local-tools/test-tools/tests - pip install local-tools/ - !reference [.run-pytest, script] tags: - - public-runner-docker + - data-cache-storage rc_direct_delegations_tests: stage: test @@ -1184,6 +1187,61 @@ authority_tests: tags: - public-runner-docker +generate_testing_block_logs_and_save_image: + stage: build + extends: [.extended_block_log_creation, .test_tools_based] + timeout: 1h + variables: + GIT_DEPTH: 0 + TESTING_BLOCK_LOGS_DIR: "$CI_PROJECT_DIR/testing_block_logs" + script: + - $CI_PROJECT_DIR/scripts/ci-helpers/testing_block_log_image_generator.sh + artifacts: + reports: + dotenv: + - testing_block_log_latest_version.env + - testing_block_log_latest_commit_short_sha.env + tags: + - public-runner-docker + - hived-for-tests + +copy_testing_block_logs_to_cache: + stage: build + extends: .docker_image_builder_job + needs: + - job: generate_testing_block_logs_and_save_image + artifacts: true + variables: + TESTING_BLOCK_LOGS_CACHE_DIR: "/cache/testing_block_logs_${TESTING_BLOCK_LOG_LATEST_COMMIT_SHORT_SHA}" + script: + - echo $TESTING_BLOCK_LOGS_CACHE_DIR + - echo "TESTING_BLOCK_LOGS_CACHE_DIR=$TESTING_BLOCK_LOGS_CACHE_DIR" > $CI_PROJECT_DIR/testing_block_log_cache_dir.env + - | + if [ -d "$TESTING_BLOCK_LOGS_CACHE_DIR" ]; then + echo "Directory $TESTING_BLOCK_LOGS_CACHE_DIR exists. Skipping copy from image process..." + else + echo "Start copy testing block logs from image - $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE" + mkdir -p $TESTING_BLOCK_LOGS_CACHE_DIR && cd $TESTING_BLOCK_LOGS_CACHE_DIR + echo "Start copy block_logs from - $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE" + docker pull $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE + $CI_PROJECT_DIR/scripts/ci-helpers/export-data-from-docker-image.sh $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE $TESTING_BLOCK_LOGS_CACHE_DIR --image-path=/ + checksum_from_image=$(docker inspect --format='{{ index .Config.Labels "testing_block_logs_checksum" }}' $TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE) + echo "Checksum from image: $checksum_from_image" + checksum_after_copy=$(find $TESTING_BLOCK_LOGS_CACHE_DIR -type f | sort | xargs cat | md5sum |cut -d ' ' -f 1) + echo "Checksum after copy: $checksum_after_copy" + if [[ "$checksum_from_image" != "$checksum_after_copy" ]]; then + echo "Checksum mismatch detected! The checksum from the container does not match the checksum after copying." + exit 1 + fi + fi + - tree $TESTING_BLOCK_LOGS_CACHE_DIR + artifacts: + reports: + dotenv: + - testing_block_log_cache_dir.env + tags: + - data-cache-storage + generate_universal_block_logs_and_save_image: stage: build extends: [.extended_block_log_creation, .test_tools_based] diff --git a/scripts/ci-helpers/testing_block_log_image_generator.sh b/scripts/ci-helpers/testing_block_log_image_generator.sh new file mode 100755 index 0000000000..3e199c6cdc --- /dev/null +++ b/scripts/ci-helpers/testing_block_log_image_generator.sh @@ -0,0 +1,70 @@ +#! /bin/bash + +set -euo pipefail + +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# shellcheck source=./docker_image_utils.sh +source "$SCRIPTPATH/docker_image_utils.sh" + +submodule_path=$CI_PROJECT_DIR +REGISTRY=$CI_REGISTRY_IMAGE +REGISTRY_USER=$REGISTRY_USER +REGISTRY_PASSWORD=$REGISTRY_PASS +IMGNAME=/testing-block-logs + +echo "Attempting to get commit for: $submodule_path" + +CHANGES=(tests/python/functional/util/testing_block_logs/generate_testing_block_logs.py) +commit=$("$SCRIPTPATH/retrieve_last_commit.sh" "${submodule_path}" "${CHANGES[@]}") +echo "commit with last source code changes is $commit" + +pushd "${submodule_path}" +short_commit=$(git -c core.abbrev=8 rev-parse --short "$commit") +popd + +prefix_tag="testing-block-log" +tag=$prefix_tag-$short_commit + +img=$( build_image_name $IMGNAME "$tag" "$REGISTRY" ) +img_path=$( build_image_registry_path $IMGNAME "$tag" "$REGISTRY" ) +img_tag=$( build_image_registry_tag $IMGNAME "$tag" "$REGISTRY" ) + +echo "$REGISTRY_PASSWORD" | docker login -u "$REGISTRY_USER" "$REGISTRY" --password-stdin + +image_exists=0 +docker_image_exists $IMGNAME "$tag" "$REGISTRY" image_exists + +if [ "$image_exists" -eq 1 ]; +then + echo "Testing block log image is up to date. Skipping generation..." +else + echo "${img} image is missing. Starting generation of testing block logs..." + echo "Save block logs to directory: $TESTING_BLOCK_LOGS_DIR" + cd tests/python/functional/util/testing_block_logs + python3 generate_testing_block_logs.py --output-block-log-directory="$TESTING_BLOCK_LOGS_DIR/block_logs_for_testing" + echo "Block logs saved in: $TESTING_BLOCK_LOGS_DIR" + + checksum=$(find $TESTING_BLOCK_LOGS_DIR -type f | sort | xargs cat | md5sum |cut -d ' ' -f 1) + echo "Checksum of the generated testing block logs: $checksum" + + echo "Build a Dockerfile" + + pwd + cd $TESTING_BLOCK_LOGS_DIR + pwd + + cat < Dockerfile +FROM scratch +LABEL testing_block_logs_checksum=${checksum} +COPY block_logs_for_testing /testing_block_logs +EOF + cat Dockerfile + echo "Build docker image containing testing_block_logs" + docker build -t $img . + docker push $img + echo "Created and push docker image with testing block logs: $img" +fi + +echo "TESTING_BLOCK_LOG_LATEST_VERSION_IMAGE=$img" > $CI_PROJECT_DIR/testing_block_log_latest_version.env +echo "TESTING_BLOCK_LOG_LATEST_COMMIT_SHORT_SHA=$short_commit" > $CI_PROJECT_DIR/testing_block_log_latest_commit_short_sha.env diff --git a/tests/python/functional/util/testing_block_logs/__init__.py b/tests/python/functional/util/testing_block_logs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/python/functional/util/testing_block_logs/generate_testing_block_logs.py b/tests/python/functional/util/testing_block_logs/generate_testing_block_logs.py new file mode 100644 index 0000000000..8d59e40ad7 --- /dev/null +++ b/tests/python/functional/util/testing_block_logs/generate_testing_block_logs.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import argparse +from pathlib import Path + +from loguru import logger + +import test_tools as tt + + +def generate_and_copy_empty_log( + split_value: int, + block_count: int, + target_dir: Path, +) -> None: + node = tt.InitNode() + node.config.block_log_split = split_value + + node.run(time_control=tt.SpeedUpRateTimeControl(speed_up_rate=10)) + tt.logger.info(f"Waiting for block {block_count}...") + node.wait_for_block_with_number(block_count) + + node.close() + node.block_log.copy_to(target_dir / f"_{block_count}", artifacts="required") + + +def prepare_empty_logs( + output_block_log_directory: Path, +) -> None: + """ + This script generates block logs (both split and monlithic) of different length with empty blocks only + """ + logger.disable("helpy") + logger.disable("test_tools") + + # Initial 30 blocks are needed to have at least 10 irreversible ones. + block_log_directory_30 = output_block_log_directory / "empty_30" + block_log_directory_30.mkdir(parents=True, exist_ok=True) + generate_and_copy_empty_log(9999, 30, block_log_directory_30) + generate_and_copy_empty_log(-1, 30, block_log_directory_30) + + # 430 blocks is enough for the split block log to be in 2 parts. + block_log_directory_430 = output_block_log_directory / "empty_430" + block_log_directory_430.mkdir(parents=True, exist_ok=True) + generate_and_copy_empty_log(9999, 430, block_log_directory_430) + generate_and_copy_empty_log(-1, 430, block_log_directory_430) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output-block-log-directory", type=Path, default=Path(__file__).parent) + args = parser.parse_args() + prepare_empty_logs(args.output_block_log_directory) + # TODO: Add generation of other logs here. diff --git a/tests/python/hive-local-tools/test-tools b/tests/python/hive-local-tools/test-tools index 37a58a6296..ea1b4c6678 160000 --- a/tests/python/hive-local-tools/test-tools +++ b/tests/python/hive-local-tools/test-tools @@ -1 +1 @@ -Subproject commit 37a58a629654b1d2e70e7ce9078628c42e595109 +Subproject commit ea1b4c6678cc4cd0442f6c86719b2989bf775d5e