From f7c0ebc52689bdb376312bbdfa11f15dd1f10685 Mon Sep 17 00:00:00 2001 From: njlr Date: Sat, 21 Sep 2024 14:53:45 +0100 Subject: [PATCH 1/8] Add example using local tar file as build context --- MODULE.bazel | 1 + WORKSPACE | 13 +++ examples/dockerfile_tar_context/BUILD.bazel | 95 +++++++++++++++++++ examples/dockerfile_tar_context/Dockerfile | 15 +++ examples/dockerfile_tar_context/README.md | 21 ++++ examples/dockerfile_tar_context/buildx.bzl | 72 ++++++++++++++ .../dockerfile_tar_context/buildx_wrapper.sh | 12 +++ .../dockerfile_tar_context/requirements.txt | 1 + examples/dockerfile_tar_context/src/say.py | 3 + 9 files changed, 233 insertions(+) create mode 100644 examples/dockerfile_tar_context/BUILD.bazel create mode 100644 examples/dockerfile_tar_context/Dockerfile create mode 100644 examples/dockerfile_tar_context/README.md create mode 100644 examples/dockerfile_tar_context/buildx.bzl create mode 100755 examples/dockerfile_tar_context/buildx_wrapper.sh create mode 100644 examples/dockerfile_tar_context/requirements.txt create mode 100644 examples/dockerfile_tar_context/src/say.py diff --git a/MODULE.bazel b/MODULE.bazel index 37eba0dc..0b284971 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -35,3 +35,4 @@ bazel_dep(name = "rules_go", version = "0.46.0", dev_dependency = True, repo_nam bazel_dep(name = "gazelle", version = "0.35.0", dev_dependency = True, repo_name = "bazel_gazelle") bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.4.1", dev_dependency = True) bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True) diff --git a/WORKSPACE b/WORKSPACE index 8708aa9d..2e3ddd60 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -61,3 +61,16 @@ load(":fetch.bzl", "fetch_images", "fetch_test_repos") fetch_images() fetch_test_repos() + +# Setup rules_pkg +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "rules_pkg", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/1.0.1/rules_pkg-1.0.1.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/1.0.1/rules_pkg-1.0.1.tar.gz", + ], + sha256 = "d20c951960ed77cb7b341c2a59488534e494d5ad1d30c4818c736d57772a9fef", +) +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") +rules_pkg_dependencies() diff --git a/examples/dockerfile_tar_context/BUILD.bazel b/examples/dockerfile_tar_context/BUILD.bazel new file mode 100644 index 00000000..b909800f --- /dev/null +++ b/examples/dockerfile_tar_context/BUILD.bazel @@ -0,0 +1,95 @@ +load("@aspect_bazel_lib//lib:run_binary.bzl", "run_binary") +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") +load("@configure_buildx//:defs.bzl", "BUILDER_NAME", "TARGET_COMPATIBLE_WITH") +load("@rules_oci//oci:defs.bzl", "oci_image") +load("//examples:assert.bzl", "assert_oci_config", "assert_oci_image_command") +load("@rules_pkg//:mappings.bzl", "strip_prefix") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") + +native_binary( + name = "buildx", + src = select({ + "@bazel_tools//src/conditions:linux_x86_64": "@buildx_linux_amd64//file", + "@bazel_tools//src/conditions:darwin_arm64": "@buildx_darwin_arm64//file", + "@bazel_tools//src/conditions:darwin_x86_64": "@buildx_darwin_amd64//file", + }), + out = "buildx", +) + +# This tar can be arranged in arbitrary ways. +# For example, one could grab pkg_files from elsewhere in the workspace. +# Anything in the tar can be used by a Dockerfile ADD or COPY. +pkg_tar( + name = "buildx_context", + srcs = [ + "Dockerfile", + "requirements.txt", + ] + glob([ + "src/*", + ]), + strip_prefix = strip_prefix.from_pkg("."), + package_dir = "/app", +) + +# In order to use a local tar as a context, we must pipe the tar into the BuildX binary +# This is a bash operation, so we must wrap the BuildX tool in an sh_binary that does this for us +sh_binary( + name = "buildx_wrapper", + srcs = [ + "buildx_wrapper.sh", + ], +) + +run_binary( + name = "base", + tool = ":buildx_wrapper", + out_dirs = [ "base" ], + srcs = [ + ":buildx_context", + ":buildx", + ], + env = { + "BUILDX": "$(location :buildx)", + "BUILDER_NAME": BUILDER_NAME, + "OUTPUT_DIR": "$@", + "BUILDX_CONTEXT": "$(location :buildx_context)", + "DOCKER_FILE": "/app/Dockerfile", + "PLATFORM": "linux/amd64", + }, + execution_requirements = { "local": "1" }, + target_compatible_with = TARGET_COMPATIBLE_WITH, +) + +oci_image( + name = "image", + base = ":base", +) + +assert_oci_config( + name = "assert_metadata", + cmd_eq = ["/app/say.py"], + entrypoint_eq = None, + image = ":image", +) + +assert_oci_image_command( + name = "assert_cow_says_moo", + args = [ + "python", + "/app/src/say.py", + ], + exit_code_eq = 0, + image = ":image", + output_eq = """\ + ____ +| moo! | + ==== + \\ + \\ + ^__^ + (oo)\\_______ + (__)\\ )\\/\\ + ||----w | + || || +""", +) diff --git a/examples/dockerfile_tar_context/Dockerfile b/examples/dockerfile_tar_context/Dockerfile new file mode 100644 index 00000000..2087538e --- /dev/null +++ b/examples/dockerfile_tar_context/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11.9-bullseye@sha256:64da8e5fd98057b05db01b49289b774e9fa3b81e87b4883079f6c31fb141b252 + +ARG DEBIAN_FRONTEND=noninteractive + +ENV LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + LANGUAGE=C.UTF-8 + +ADD /app /app + +WORKDIR /app + +RUN pip install --root-user-action=ignore -r requirements.txt + +CMD ["/app/say.py"] \ No newline at end of file diff --git a/examples/dockerfile_tar_context/README.md b/examples/dockerfile_tar_context/README.md new file mode 100644 index 00000000..221b11fa --- /dev/null +++ b/examples/dockerfile_tar_context/README.md @@ -0,0 +1,21 @@ +# Dockerfile + rules_oci (tar context) + +STOP before committing this atrocity. Here's some good reasons why you should not do what we have done here. + +- Dockerfiles are fundamentally non-reproducible +- Reproducible builds are important for Bazel, Dockerfiles will lead to poor cache hits. +- `RUN` instruction is a perfect foot-gun for non-reprocubile builds, a simple command `RUN apt-get install curl` is non-hermetic by default. +- Building the same Dockerfile one month apart will yield different results. +- `FROM python:3.11.9-bullseye` is non-producible. + +So you have chosen to walk this path... one thing that can be tricky is to get Bazel artifacts into the build context of Docker. Docker does not like symlinks or backtracking out of the build context. However, we can provide a complete context using [a tar file and piping it into BuildX](https://docs.docker.com/build/concepts/context/#local-tarballs). Since we have full control over tar layout in Bazel, we now have full control over the build context! + +This technique offers an easy migration path from `container_run_and_commit` in `rules_docker`, but use it sparingly. + +# Resources + +- https://reproducible-builds.org/ +- https://github.com/bazel-contrib/rules_oci/issues/35#issuecomment-1285954483 +- https://github.com/bazel-contrib/rules_oci/blob/main/docs/compare_dockerfile.md +- https://github.com/moby/moby/issues/43124 +- https://medium.com/nttlabs/bit-for-bit-reproducible-builds-with-dockerfile-7cc2b9faed9f diff --git a/examples/dockerfile_tar_context/buildx.bzl b/examples/dockerfile_tar_context/buildx.bzl new file mode 100644 index 00000000..20b7d6ec --- /dev/null +++ b/examples/dockerfile_tar_context/buildx.bzl @@ -0,0 +1,72 @@ +"repos for buildx" + +load("@aspect_bazel_lib//lib:repo_utils.bzl", "repo_utils") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + +def _impl_configure_buildx(rctx): + has_docker = False + # See if standard docker sock exists + if not has_docker: + r = rctx.execute(["stat", "/var/run/docker.sock"]) + if r.return_code == 0: + has_docker = True + + compatible_with = "[]" + builder_name = "builder-docker" + if has_docker: + buildx = rctx.path(Label("@buildx_%s//file:downloaded" % repo_utils.platform(rctx))) + + r = rctx.execute([buildx, "ls"]) + if not builder_name in r.stdout: + r = rctx.execute([buildx, "create", "--name", builder_name, "--driver", "docker-container"]) + if r.return_code != 0: + fail("Failed to create buildx driver %s: \nSTDERR:\n%s\nsSTDOUT:\n%s" % (builder_name, r.stderr, r.stdout)) + else: + # buildifier: disable=print + print("WARNING: BuildX driver `%s` already exists." % builder_name) + + else: + compatible_with = '["@platforms//:incompatible"]' + + rctx.file("defs.bzl", """ +# Generated by configure_buildx.bzl +TARGET_COMPATIBLE_WITH = %s +BUILDER_NAME = "%s" +""" % (compatible_with, builder_name)) + rctx.file("BUILD.bazel", 'exports_files(["defs.bzl"])') + pass + +configure_buildx = repository_rule( + implementation = _impl_configure_buildx, + local = True, +) + +def fetch_buildx(): + http_file( + name = "buildx_linux_amd64", + urls = [ + "https://github.com/docker/buildx/releases/download/v0.14.0/buildx-v0.14.0.linux-amd64", + ], + integrity = "sha256-Mvjxfso1vy7+bA5H9A5Gkqh280UxtCHvyYR5mltBIm4=", + executable = True, + ) + + http_file( + name = "buildx_darwin_arm64", + urls = [ + "https://github.com/docker/buildx/releases/download/v0.14.0/buildx-v0.14.0.darwin-arm64", + ], + integrity = "sha256-3BdvI2ZgnMITKubwi7IZOjL5/ZNUv9Agz3+juNt0hA0=", + executable = True, + ) + + http_file( + name = "buildx_darwin_amd64", + urls = [ + "https://github.com/docker/buildx/releases/download/v0.14.0/buildx-v0.14.0.darwin-amd64", + ], + integrity = "sha256-J6rZfENSvCzFBHDgnA8Oqq2FDXR+M9CTejhhg9DruPU=", + executable = True, + ) + + configure_buildx(name = "configure_buildx") diff --git a/examples/dockerfile_tar_context/buildx_wrapper.sh b/examples/dockerfile_tar_context/buildx_wrapper.sh new file mode 100755 index 00000000..e35c5bed --- /dev/null +++ b/examples/dockerfile_tar_context/buildx_wrapper.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail + +$BUILDX \ + build - \ + --no-cache \ + --builder $BUILDER_NAME \ + --file $DOCKER_FILE \ + --platform $PLATFORM \ + --output=type=oci,tar=false,dest=$OUTPUT_DIR \ + < $BUILDX_CONTEXT diff --git a/examples/dockerfile_tar_context/requirements.txt b/examples/dockerfile_tar_context/requirements.txt new file mode 100644 index 00000000..9c405168 --- /dev/null +++ b/examples/dockerfile_tar_context/requirements.txt @@ -0,0 +1 @@ +cowsay==6.1.0 \ No newline at end of file diff --git a/examples/dockerfile_tar_context/src/say.py b/examples/dockerfile_tar_context/src/say.py new file mode 100644 index 00000000..6f19226d --- /dev/null +++ b/examples/dockerfile_tar_context/src/say.py @@ -0,0 +1,3 @@ +import cowsay + +cowsay.cow('moo!') \ No newline at end of file From 44ccebf72a15dcb33a36bfb42a2c9bca9c1ca423 Mon Sep 17 00:00:00 2001 From: njlr Date: Wed, 16 Oct 2024 18:05:14 +0100 Subject: [PATCH 2/8] Merge examples --- examples/dockerfile/BUILD.bazel | 76 ++++++++------- examples/dockerfile/Dockerfile | 11 +-- examples/dockerfile/README.md | 16 ++-- .../buildx_wrapper.sh | 0 .../requirements.txt | 0 examples/dockerfile_tar_context/BUILD.bazel | 95 ------------------- examples/dockerfile_tar_context/Dockerfile | 15 --- examples/dockerfile_tar_context/README.md | 21 ---- examples/dockerfile_tar_context/buildx.bzl | 72 -------------- examples/dockerfile_tar_context/src/say.py | 3 - 10 files changed, 55 insertions(+), 254 deletions(-) rename examples/{dockerfile_tar_context => dockerfile}/buildx_wrapper.sh (100%) rename examples/{dockerfile_tar_context => dockerfile}/requirements.txt (100%) delete mode 100644 examples/dockerfile_tar_context/BUILD.bazel delete mode 100644 examples/dockerfile_tar_context/Dockerfile delete mode 100644 examples/dockerfile_tar_context/README.md delete mode 100644 examples/dockerfile_tar_context/buildx.bzl delete mode 100644 examples/dockerfile_tar_context/src/say.py diff --git a/examples/dockerfile/BUILD.bazel b/examples/dockerfile/BUILD.bazel index 887476d4..b909800f 100644 --- a/examples/dockerfile/BUILD.bazel +++ b/examples/dockerfile/BUILD.bazel @@ -3,6 +3,8 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@configure_buildx//:defs.bzl", "BUILDER_NAME", "TARGET_COMPATIBLE_WITH") load("@rules_oci//oci:defs.bzl", "oci_image") load("//examples:assert.bzl", "assert_oci_config", "assert_oci_image_command") +load("@rules_pkg//:mappings.bzl", "strip_prefix") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") native_binary( name = "buildx", @@ -14,22 +16,48 @@ native_binary( out = "buildx", ) -# docker buildx create --name container --driver=docker-container +# This tar can be arranged in arbitrary ways. +# For example, one could grab pkg_files from elsewhere in the workspace. +# Anything in the tar can be used by a Dockerfile ADD or COPY. +pkg_tar( + name = "buildx_context", + srcs = [ + "Dockerfile", + "requirements.txt", + ] + glob([ + "src/*", + ]), + strip_prefix = strip_prefix.from_pkg("."), + package_dir = "/app", +) + +# In order to use a local tar as a context, we must pipe the tar into the BuildX binary +# This is a bash operation, so we must wrap the BuildX tool in an sh_binary that does this for us +sh_binary( + name = "buildx_wrapper", + srcs = [ + "buildx_wrapper.sh", + ], +) + run_binary( name = "base", - srcs = ["Dockerfile"] + glob(["src/*"]), - args = [ - "build", - "./examples/dockerfile", - "--builder", - BUILDER_NAME, - "--output=type=oci,tar=false,dest=$@", + tool = ":buildx_wrapper", + out_dirs = [ "base" ], + srcs = [ + ":buildx_context", + ":buildx", ], - execution_requirements = {"local": "1"}, - mnemonic = "BuildDocker", - out_dirs = ["base"], + env = { + "BUILDX": "$(location :buildx)", + "BUILDER_NAME": BUILDER_NAME, + "OUTPUT_DIR": "$@", + "BUILDX_CONTEXT": "$(location :buildx_context)", + "DOCKER_FILE": "/app/Dockerfile", + "PLATFORM": "linux/amd64", + }, + execution_requirements = { "local": "1" }, target_compatible_with = TARGET_COMPATIBLE_WITH, - tool = ":buildx", ) oci_image( @@ -44,33 +72,11 @@ assert_oci_config( image = ":image", ) -assert_oci_image_command( - name = "assert_jq_works", - args = [ - "jq", - "--version", - ], - exit_code_eq = 0, - image = ":image", - output_eq = "jq-1.6\n", -) - -assert_oci_image_command( - name = "assert_apt_lists_still_exist", - args = [ - "file", - "/var/lib/apt/lists", - ], - exit_code_eq = 0, - image = ":image", - output_eq = "/var/lib/apt/lists: directory\n", -) - assert_oci_image_command( name = "assert_cow_says_moo", args = [ "python", - "/app/say.py", + "/app/src/say.py", ], exit_code_eq = 0, image = ":image", diff --git a/examples/dockerfile/Dockerfile b/examples/dockerfile/Dockerfile index a4a7c862..2087538e 100644 --- a/examples/dockerfile/Dockerfile +++ b/examples/dockerfile/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.9-bullseye +FROM python:3.11.9-bullseye@sha256:64da8e5fd98057b05db01b49289b774e9fa3b81e87b4883079f6c31fb141b252 ARG DEBIAN_FRONTEND=noninteractive @@ -6,13 +6,10 @@ ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 \ LANGUAGE=C.UTF-8 -RUN apt-get -y update \ - && apt-get -y install jq \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +ADD /app /app -RUN pip install cowsay +WORKDIR /app -COPY src /app +RUN pip install --root-user-action=ignore -r requirements.txt CMD ["/app/say.py"] \ No newline at end of file diff --git a/examples/dockerfile/README.md b/examples/dockerfile/README.md index c2bb39a0..221b11fa 100644 --- a/examples/dockerfile/README.md +++ b/examples/dockerfile/README.md @@ -1,4 +1,4 @@ -# Dockerfile + rules_oci +# Dockerfile + rules_oci (tar context) STOP before committing this atrocity. Here's some good reasons why you should not do what we have done here. @@ -8,10 +8,14 @@ STOP before committing this atrocity. Here's some good reasons why you should no - Building the same Dockerfile one month apart will yield different results. - `FROM python:3.11.9-bullseye` is non-producible. +So you have chosen to walk this path... one thing that can be tricky is to get Bazel artifacts into the build context of Docker. Docker does not like symlinks or backtracking out of the build context. However, we can provide a complete context using [a tar file and piping it into BuildX](https://docs.docker.com/build/concepts/context/#local-tarballs). Since we have full control over tar layout in Bazel, we now have full control over the build context! + +This technique offers an easy migration path from `container_run_and_commit` in `rules_docker`, but use it sparingly. + # Resources -https://reproducible-builds.org/ -https://github.com/bazel-contrib/rules_oci/issues/35#issuecomment-1285954483 -https://github.com/bazel-contrib/rules_oci/blob/main/docs/compare_dockerfile.md -https://github.com/moby/moby/issues/43124 -https://medium.com/nttlabs/bit-for-bit-reproducible-builds-with-dockerfile-7cc2b9faed9f +- https://reproducible-builds.org/ +- https://github.com/bazel-contrib/rules_oci/issues/35#issuecomment-1285954483 +- https://github.com/bazel-contrib/rules_oci/blob/main/docs/compare_dockerfile.md +- https://github.com/moby/moby/issues/43124 +- https://medium.com/nttlabs/bit-for-bit-reproducible-builds-with-dockerfile-7cc2b9faed9f diff --git a/examples/dockerfile_tar_context/buildx_wrapper.sh b/examples/dockerfile/buildx_wrapper.sh similarity index 100% rename from examples/dockerfile_tar_context/buildx_wrapper.sh rename to examples/dockerfile/buildx_wrapper.sh diff --git a/examples/dockerfile_tar_context/requirements.txt b/examples/dockerfile/requirements.txt similarity index 100% rename from examples/dockerfile_tar_context/requirements.txt rename to examples/dockerfile/requirements.txt diff --git a/examples/dockerfile_tar_context/BUILD.bazel b/examples/dockerfile_tar_context/BUILD.bazel deleted file mode 100644 index b909800f..00000000 --- a/examples/dockerfile_tar_context/BUILD.bazel +++ /dev/null @@ -1,95 +0,0 @@ -load("@aspect_bazel_lib//lib:run_binary.bzl", "run_binary") -load("@bazel_skylib//rules:native_binary.bzl", "native_binary") -load("@configure_buildx//:defs.bzl", "BUILDER_NAME", "TARGET_COMPATIBLE_WITH") -load("@rules_oci//oci:defs.bzl", "oci_image") -load("//examples:assert.bzl", "assert_oci_config", "assert_oci_image_command") -load("@rules_pkg//:mappings.bzl", "strip_prefix") -load("@rules_pkg//pkg:tar.bzl", "pkg_tar") - -native_binary( - name = "buildx", - src = select({ - "@bazel_tools//src/conditions:linux_x86_64": "@buildx_linux_amd64//file", - "@bazel_tools//src/conditions:darwin_arm64": "@buildx_darwin_arm64//file", - "@bazel_tools//src/conditions:darwin_x86_64": "@buildx_darwin_amd64//file", - }), - out = "buildx", -) - -# This tar can be arranged in arbitrary ways. -# For example, one could grab pkg_files from elsewhere in the workspace. -# Anything in the tar can be used by a Dockerfile ADD or COPY. -pkg_tar( - name = "buildx_context", - srcs = [ - "Dockerfile", - "requirements.txt", - ] + glob([ - "src/*", - ]), - strip_prefix = strip_prefix.from_pkg("."), - package_dir = "/app", -) - -# In order to use a local tar as a context, we must pipe the tar into the BuildX binary -# This is a bash operation, so we must wrap the BuildX tool in an sh_binary that does this for us -sh_binary( - name = "buildx_wrapper", - srcs = [ - "buildx_wrapper.sh", - ], -) - -run_binary( - name = "base", - tool = ":buildx_wrapper", - out_dirs = [ "base" ], - srcs = [ - ":buildx_context", - ":buildx", - ], - env = { - "BUILDX": "$(location :buildx)", - "BUILDER_NAME": BUILDER_NAME, - "OUTPUT_DIR": "$@", - "BUILDX_CONTEXT": "$(location :buildx_context)", - "DOCKER_FILE": "/app/Dockerfile", - "PLATFORM": "linux/amd64", - }, - execution_requirements = { "local": "1" }, - target_compatible_with = TARGET_COMPATIBLE_WITH, -) - -oci_image( - name = "image", - base = ":base", -) - -assert_oci_config( - name = "assert_metadata", - cmd_eq = ["/app/say.py"], - entrypoint_eq = None, - image = ":image", -) - -assert_oci_image_command( - name = "assert_cow_says_moo", - args = [ - "python", - "/app/src/say.py", - ], - exit_code_eq = 0, - image = ":image", - output_eq = """\ - ____ -| moo! | - ==== - \\ - \\ - ^__^ - (oo)\\_______ - (__)\\ )\\/\\ - ||----w | - || || -""", -) diff --git a/examples/dockerfile_tar_context/Dockerfile b/examples/dockerfile_tar_context/Dockerfile deleted file mode 100644 index 2087538e..00000000 --- a/examples/dockerfile_tar_context/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.11.9-bullseye@sha256:64da8e5fd98057b05db01b49289b774e9fa3b81e87b4883079f6c31fb141b252 - -ARG DEBIAN_FRONTEND=noninteractive - -ENV LC_ALL=C.UTF-8 \ - LANG=C.UTF-8 \ - LANGUAGE=C.UTF-8 - -ADD /app /app - -WORKDIR /app - -RUN pip install --root-user-action=ignore -r requirements.txt - -CMD ["/app/say.py"] \ No newline at end of file diff --git a/examples/dockerfile_tar_context/README.md b/examples/dockerfile_tar_context/README.md deleted file mode 100644 index 221b11fa..00000000 --- a/examples/dockerfile_tar_context/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Dockerfile + rules_oci (tar context) - -STOP before committing this atrocity. Here's some good reasons why you should not do what we have done here. - -- Dockerfiles are fundamentally non-reproducible -- Reproducible builds are important for Bazel, Dockerfiles will lead to poor cache hits. -- `RUN` instruction is a perfect foot-gun for non-reprocubile builds, a simple command `RUN apt-get install curl` is non-hermetic by default. -- Building the same Dockerfile one month apart will yield different results. -- `FROM python:3.11.9-bullseye` is non-producible. - -So you have chosen to walk this path... one thing that can be tricky is to get Bazel artifacts into the build context of Docker. Docker does not like symlinks or backtracking out of the build context. However, we can provide a complete context using [a tar file and piping it into BuildX](https://docs.docker.com/build/concepts/context/#local-tarballs). Since we have full control over tar layout in Bazel, we now have full control over the build context! - -This technique offers an easy migration path from `container_run_and_commit` in `rules_docker`, but use it sparingly. - -# Resources - -- https://reproducible-builds.org/ -- https://github.com/bazel-contrib/rules_oci/issues/35#issuecomment-1285954483 -- https://github.com/bazel-contrib/rules_oci/blob/main/docs/compare_dockerfile.md -- https://github.com/moby/moby/issues/43124 -- https://medium.com/nttlabs/bit-for-bit-reproducible-builds-with-dockerfile-7cc2b9faed9f diff --git a/examples/dockerfile_tar_context/buildx.bzl b/examples/dockerfile_tar_context/buildx.bzl deleted file mode 100644 index 20b7d6ec..00000000 --- a/examples/dockerfile_tar_context/buildx.bzl +++ /dev/null @@ -1,72 +0,0 @@ -"repos for buildx" - -load("@aspect_bazel_lib//lib:repo_utils.bzl", "repo_utils") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") - -def _impl_configure_buildx(rctx): - has_docker = False - # See if standard docker sock exists - if not has_docker: - r = rctx.execute(["stat", "/var/run/docker.sock"]) - if r.return_code == 0: - has_docker = True - - compatible_with = "[]" - builder_name = "builder-docker" - if has_docker: - buildx = rctx.path(Label("@buildx_%s//file:downloaded" % repo_utils.platform(rctx))) - - r = rctx.execute([buildx, "ls"]) - if not builder_name in r.stdout: - r = rctx.execute([buildx, "create", "--name", builder_name, "--driver", "docker-container"]) - if r.return_code != 0: - fail("Failed to create buildx driver %s: \nSTDERR:\n%s\nsSTDOUT:\n%s" % (builder_name, r.stderr, r.stdout)) - else: - # buildifier: disable=print - print("WARNING: BuildX driver `%s` already exists." % builder_name) - - else: - compatible_with = '["@platforms//:incompatible"]' - - rctx.file("defs.bzl", """ -# Generated by configure_buildx.bzl -TARGET_COMPATIBLE_WITH = %s -BUILDER_NAME = "%s" -""" % (compatible_with, builder_name)) - rctx.file("BUILD.bazel", 'exports_files(["defs.bzl"])') - pass - -configure_buildx = repository_rule( - implementation = _impl_configure_buildx, - local = True, -) - -def fetch_buildx(): - http_file( - name = "buildx_linux_amd64", - urls = [ - "https://github.com/docker/buildx/releases/download/v0.14.0/buildx-v0.14.0.linux-amd64", - ], - integrity = "sha256-Mvjxfso1vy7+bA5H9A5Gkqh280UxtCHvyYR5mltBIm4=", - executable = True, - ) - - http_file( - name = "buildx_darwin_arm64", - urls = [ - "https://github.com/docker/buildx/releases/download/v0.14.0/buildx-v0.14.0.darwin-arm64", - ], - integrity = "sha256-3BdvI2ZgnMITKubwi7IZOjL5/ZNUv9Agz3+juNt0hA0=", - executable = True, - ) - - http_file( - name = "buildx_darwin_amd64", - urls = [ - "https://github.com/docker/buildx/releases/download/v0.14.0/buildx-v0.14.0.darwin-amd64", - ], - integrity = "sha256-J6rZfENSvCzFBHDgnA8Oqq2FDXR+M9CTejhhg9DruPU=", - executable = True, - ) - - configure_buildx(name = "configure_buildx") diff --git a/examples/dockerfile_tar_context/src/say.py b/examples/dockerfile_tar_context/src/say.py deleted file mode 100644 index 6f19226d..00000000 --- a/examples/dockerfile_tar_context/src/say.py +++ /dev/null @@ -1,3 +0,0 @@ -import cowsay - -cowsay.cow('moo!') \ No newline at end of file From 7795f196a58e7a12c3197bc16615e2eba1293f0c Mon Sep 17 00:00:00 2001 From: njlr Date: Wed, 16 Oct 2024 18:10:38 +0100 Subject: [PATCH 3/8] Restore jq tests --- examples/dockerfile/BUILD.bazel | 27 ++++++++++++++++++++++++++- examples/dockerfile/Dockerfile | 5 +++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/examples/dockerfile/BUILD.bazel b/examples/dockerfile/BUILD.bazel index b909800f..377c5ab4 100644 --- a/examples/dockerfile/BUILD.bazel +++ b/examples/dockerfile/BUILD.bazel @@ -56,7 +56,10 @@ run_binary( "DOCKER_FILE": "/app/Dockerfile", "PLATFORM": "linux/amd64", }, - execution_requirements = { "local": "1" }, + execution_requirements = { + "local": "1", + "requires-network": "1", + }, target_compatible_with = TARGET_COMPATIBLE_WITH, ) @@ -65,6 +68,28 @@ oci_image( base = ":base", ) +assert_oci_image_command( + name = "assert_jq_works", + args = [ + "jq", + "--version", + ], + exit_code_eq = 0, + image = ":image", + output_eq = "jq-1.6\n", +) + +assert_oci_image_command( + name = "assert_apt_lists_still_exist", + args = [ + "file", + "/var/lib/apt/lists", + ], + exit_code_eq = 0, + image = ":image", + output_eq = "/var/lib/apt/lists: directory\n", +) + assert_oci_config( name = "assert_metadata", cmd_eq = ["/app/say.py"], diff --git a/examples/dockerfile/Dockerfile b/examples/dockerfile/Dockerfile index 2087538e..0d05cf96 100644 --- a/examples/dockerfile/Dockerfile +++ b/examples/dockerfile/Dockerfile @@ -6,6 +6,11 @@ ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 \ LANGUAGE=C.UTF-8 +RUN apt-get -y update \ + && apt-get -y install jq \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + ADD /app /app WORKDIR /app From 831d7ac7e36bed0bac836e9d4b0ff7bdf776d531 Mon Sep 17 00:00:00 2001 From: njlr Date: Wed, 16 Oct 2024 18:12:16 +0100 Subject: [PATCH 4/8] Revert title --- examples/dockerfile/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/dockerfile/README.md b/examples/dockerfile/README.md index 221b11fa..22432280 100644 --- a/examples/dockerfile/README.md +++ b/examples/dockerfile/README.md @@ -1,4 +1,4 @@ -# Dockerfile + rules_oci (tar context) +# Dockerfile + rules_oci STOP before committing this atrocity. Here's some good reasons why you should not do what we have done here. From b2c07cce32eabd5975f72ab46e3fcdaea3736b5d Mon Sep 17 00:00:00 2001 From: njlr Date: Wed, 16 Oct 2024 18:23:13 +0100 Subject: [PATCH 5/8] Side by side tests --- examples/dockerfile/BUILD.bazel | 119 ++++++++++++++++++------- examples/dockerfile/Dockerfile | 6 +- examples/dockerfile/context.Dockerfile | 15 ++++ 3 files changed, 103 insertions(+), 37 deletions(-) create mode 100644 examples/dockerfile/context.Dockerfile diff --git a/examples/dockerfile/BUILD.bazel b/examples/dockerfile/BUILD.bazel index 377c5ab4..62109e6a 100644 --- a/examples/dockerfile/BUILD.bazel +++ b/examples/dockerfile/BUILD.bazel @@ -16,13 +16,87 @@ native_binary( out = "buildx", ) +# docker buildx create --name container --driver=docker-container +run_binary( + name = "base", + srcs = ["Dockerfile"] + glob(["src/*"]), + args = [ + "build", + "./examples/dockerfile", + "--builder", + BUILDER_NAME, + "--output=type=oci,tar=false,dest=$@", + ], + execution_requirements = {"local": "1"}, + mnemonic = "BuildDocker", + out_dirs = ["base"], + target_compatible_with = TARGET_COMPATIBLE_WITH, + tool = ":buildx", +) + +oci_image( + name = "image", + base = ":base", +) + +assert_oci_image_command( + name = "assert_jq_works", + args = [ + "jq", + "--version", + ], + exit_code_eq = 0, + image = ":image", + output_eq = "jq-1.6\n", +) + +assert_oci_image_command( + name = "assert_apt_lists_still_exist", + args = [ + "file", + "/var/lib/apt/lists", + ], + exit_code_eq = 0, + image = ":image", + output_eq = "/var/lib/apt/lists: directory\n", +) + +assert_oci_config( + name = "assert_metadata", + cmd_eq = ["/app/say.py"], + entrypoint_eq = None, + image = ":image", +) + +assert_oci_image_command( + name = "assert_cow_says_moo", + args = [ + "python", + "/app/say.py", + ], + exit_code_eq = 0, + image = ":image", + output_eq = """\ + ____ +| moo! | + ==== + \\ + \\ + ^__^ + (oo)\\_______ + (__)\\ )\\/\\ + ||----w | + || || +""", +) + # This tar can be arranged in arbitrary ways. # For example, one could grab pkg_files from elsewhere in the workspace. # Anything in the tar can be used by a Dockerfile ADD or COPY. pkg_tar( name = "buildx_context", srcs = [ - "Dockerfile", + "context.Dockerfile", "requirements.txt", ] + glob([ "src/*", @@ -40,10 +114,11 @@ sh_binary( ], ) +# This is similar to :base except it uses a tar file as a Docker context run_binary( - name = "base", + name = "base_context", tool = ":buildx_wrapper", - out_dirs = [ "base" ], + out_dirs = [ "base_context" ], srcs = [ ":buildx_context", ":buildx", @@ -53,7 +128,7 @@ run_binary( "BUILDER_NAME": BUILDER_NAME, "OUTPUT_DIR": "$@", "BUILDX_CONTEXT": "$(location :buildx_context)", - "DOCKER_FILE": "/app/Dockerfile", + "DOCKER_FILE": "/app/context.Dockerfile", "PLATFORM": "linux/amd64", }, execution_requirements = { @@ -64,47 +139,25 @@ run_binary( ) oci_image( - name = "image", - base = ":base", -) - -assert_oci_image_command( - name = "assert_jq_works", - args = [ - "jq", - "--version", - ], - exit_code_eq = 0, - image = ":image", - output_eq = "jq-1.6\n", -) - -assert_oci_image_command( - name = "assert_apt_lists_still_exist", - args = [ - "file", - "/var/lib/apt/lists", - ], - exit_code_eq = 0, - image = ":image", - output_eq = "/var/lib/apt/lists: directory\n", + name = "image_context", + base = ":base_context", ) assert_oci_config( - name = "assert_metadata", - cmd_eq = ["/app/say.py"], + name = "assert_metadata_context", + cmd_eq = ["/app/src/say.py"], entrypoint_eq = None, - image = ":image", + image = ":image_context", ) assert_oci_image_command( - name = "assert_cow_says_moo", + name = "assert_cow_says_moo_context", args = [ "python", "/app/src/say.py", ], exit_code_eq = 0, - image = ":image", + image = ":image_context", output_eq = """\ ____ | moo! | diff --git a/examples/dockerfile/Dockerfile b/examples/dockerfile/Dockerfile index 0d05cf96..85aea135 100644 --- a/examples/dockerfile/Dockerfile +++ b/examples/dockerfile/Dockerfile @@ -11,10 +11,8 @@ RUN apt-get -y update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -ADD /app /app +RUN pip install cowsay -WORKDIR /app - -RUN pip install --root-user-action=ignore -r requirements.txt +COPY src /app CMD ["/app/say.py"] \ No newline at end of file diff --git a/examples/dockerfile/context.Dockerfile b/examples/dockerfile/context.Dockerfile new file mode 100644 index 00000000..9b9ae1d5 --- /dev/null +++ b/examples/dockerfile/context.Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11.9-bullseye@sha256:64da8e5fd98057b05db01b49289b774e9fa3b81e87b4883079f6c31fb141b252 + +ARG DEBIAN_FRONTEND=noninteractive + +ENV LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 \ + LANGUAGE=C.UTF-8 + +ADD /app /app + +WORKDIR /app + +RUN pip install --root-user-action=ignore -r requirements.txt + +CMD ["/app/src/say.py"] \ No newline at end of file From 48d1f6e2756b19e18dc1b4871b5ffdeba1618cde Mon Sep 17 00:00:00 2001 From: njlr Date: Wed, 16 Oct 2024 18:24:59 +0100 Subject: [PATCH 6/8] Revert order --- examples/dockerfile/BUILD.bazel | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/dockerfile/BUILD.bazel b/examples/dockerfile/BUILD.bazel index 62109e6a..7c3444cc 100644 --- a/examples/dockerfile/BUILD.bazel +++ b/examples/dockerfile/BUILD.bazel @@ -39,6 +39,13 @@ oci_image( base = ":base", ) +assert_oci_config( + name = "assert_metadata", + cmd_eq = ["/app/say.py"], + entrypoint_eq = None, + image = ":image", +) + assert_oci_image_command( name = "assert_jq_works", args = [ @@ -61,13 +68,6 @@ assert_oci_image_command( output_eq = "/var/lib/apt/lists: directory\n", ) -assert_oci_config( - name = "assert_metadata", - cmd_eq = ["/app/say.py"], - entrypoint_eq = None, - image = ":image", -) - assert_oci_image_command( name = "assert_cow_says_moo", args = [ From 107df5906a17b22e323e0a05fdab1054dde4c9fc Mon Sep 17 00:00:00 2001 From: njlr Date: Mon, 28 Oct 2024 20:41:54 +0000 Subject: [PATCH 7/8] Switch to tar --- MODULE.bazel | 3 +-- WORKSPACE | 13 ------------- examples/dockerfile/BUILD.bazel | 33 +++++++++++++++++++++------------ 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 0b284971..69d5df0f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -6,7 +6,7 @@ module( compatibility_level = 1, ) -bazel_dep(name = "aspect_bazel_lib", version = "2.7.2") +bazel_dep(name = "aspect_bazel_lib", version = "2.9.3") bazel_dep(name = "bazel_features", version = "1.10.0") bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "platforms", version = "0.0.8") @@ -35,4 +35,3 @@ bazel_dep(name = "rules_go", version = "0.46.0", dev_dependency = True, repo_nam bazel_dep(name = "gazelle", version = "0.35.0", dev_dependency = True, repo_name = "bazel_gazelle") bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.4.1", dev_dependency = True) bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) -bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True) diff --git a/WORKSPACE b/WORKSPACE index 2e3ddd60..8708aa9d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -61,16 +61,3 @@ load(":fetch.bzl", "fetch_images", "fetch_test_repos") fetch_images() fetch_test_repos() - -# Setup rules_pkg -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "rules_pkg", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/1.0.1/rules_pkg-1.0.1.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/1.0.1/rules_pkg-1.0.1.tar.gz", - ], - sha256 = "d20c951960ed77cb7b341c2a59488534e494d5ad1d30c4818c736d57772a9fef", -) -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") -rules_pkg_dependencies() diff --git a/examples/dockerfile/BUILD.bazel b/examples/dockerfile/BUILD.bazel index 7c3444cc..1b4194dc 100644 --- a/examples/dockerfile/BUILD.bazel +++ b/examples/dockerfile/BUILD.bazel @@ -1,10 +1,9 @@ +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_mutate", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:run_binary.bzl", "run_binary") load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@configure_buildx//:defs.bzl", "BUILDER_NAME", "TARGET_COMPATIBLE_WITH") load("@rules_oci//oci:defs.bzl", "oci_image") load("//examples:assert.bzl", "assert_oci_config", "assert_oci_image_command") -load("@rules_pkg//:mappings.bzl", "strip_prefix") -load("@rules_pkg//pkg:tar.bzl", "pkg_tar") native_binary( name = "buildx", @@ -93,16 +92,26 @@ assert_oci_image_command( # This tar can be arranged in arbitrary ways. # For example, one could grab pkg_files from elsewhere in the workspace. # Anything in the tar can be used by a Dockerfile ADD or COPY. -pkg_tar( - name = "buildx_context", - srcs = [ - "context.Dockerfile", - "requirements.txt", - ] + glob([ - "src/*", - ]), - strip_prefix = strip_prefix.from_pkg("."), - package_dir = "/app", +mtree_spec( + name = "buildx_context_tree", + srcs = [ + "context.Dockerfile", + "requirements.txt", + ] + glob([ + "src/*", + ]), +) + +mtree_mutate( + name = "buildx_context_tree_prefixed", + mtree = ":buildx_context_tree", + strip_prefix = package_name(), + package_dir = "/app", +) + +tar( + name = "buildx_context", + mtree = ":buildx_context_tree_prefixed", ) # In order to use a local tar as a context, we must pipe the tar into the BuildX binary From 1a71af657f40041d26f03f5dc179385feb4370e6 Mon Sep 17 00:00:00 2001 From: njlr Date: Thu, 21 Nov 2024 17:58:04 +0000 Subject: [PATCH 8/8] Fix tar invocation --- examples/dockerfile/BUILD.bazel | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/dockerfile/BUILD.bazel b/examples/dockerfile/BUILD.bazel index 1b4194dc..54f51c71 100644 --- a/examples/dockerfile/BUILD.bazel +++ b/examples/dockerfile/BUILD.bazel @@ -106,12 +106,18 @@ mtree_mutate( name = "buildx_context_tree_prefixed", mtree = ":buildx_context_tree", strip_prefix = package_name(), - package_dir = "/app", + package_dir = "app", ) tar( name = "buildx_context", mtree = ":buildx_context_tree_prefixed", + srcs = [ + "context.Dockerfile", + "requirements.txt", + ] + glob([ + "src/*", + ]), ) # In order to use a local tar as a context, we must pipe the tar into the BuildX binary