Add Flink 1.18 support (#5)

.github/workflows/build.yml

@@ -15,8 +15,11 @@ jobs:
     strategy:
       matrix:
         include:
-          - directory: flink1.17
-            tag: 1.17.1-python3.10
+          - tag: 1.17.2-python3.10
+            args: |
+              FLINK_VERSION=1.17.2
+              PYTHON_VERSION=3.10.13
+          - tag: 1.18.1-python3.10
             args: ''
     steps:
       - name: Checkout
@@ -48,7 +51,6 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: docker/${{ matrix.directory }}/Dockerfile
           build-args: ${{ matrix.args }}
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}

docker/flink1.17/Dockerfile → Dockerfile

@@ -1,7 +1,7 @@
-ARG FLINK_VERSION=1.17.1
+ARG FLINK_VERSION=1.18.1
 ARG FLINK_SCALA_VERSION=2.12
-ARG PYTHON_PYENV_VERSION=2.3.29
-ARG PYTHON_VERSION=3.10.11
+ARG PYTHON_PYENV_VERSION=2.3.36
+ARG PYTHON_VERSION=3.10.13
 ARG MAVEN_VERSION=3.8.6
 
 FROM flink:$FLINK_VERSION AS base
@@ -73,11 +73,14 @@ RUN set -eux; \
 
 FROM maven:${MAVEN_VERSION}-amazoncorretto-8 AS tools
 
+ARG FLINK_VERSION
+
 COPY pom.xml /app/pom.xml
 COPY tools /app/tools
 
 RUN --mount=type=cache,target=/root/.m2,sharing=locked \
     cd /app && \
+    mvn versions:set -DnewVersion=${FLINK_VERSION} -DgenerateBackupPoms=false && \
     mvn package && \
     mkdir /output && \
     cp tools/flink-fs-utils/target/*.jar /output
@@ -92,4 +95,4 @@ RUN rm /opt/flink/lib/flink-table-planner-loader-${FLINK_VERSION}.jar && \
     cp /opt/flink/opt/flink-table-planner_${FLINK_SCALA_VERSION}-${FLINK_VERSION}.jar /opt/flink/lib/
 
 COPY --from=tools /output/*.jar /opt/flink/opt/
-COPY bin/flink-fs-cp-${FLINK_VERSION}.sh /opt/flink/bin/flink-fs-cp.sh
+COPY bin/flink-fs-cp.sh /opt/flink/bin/flink-fs-cp.sh

README.md

@@ -1,16 +1,18 @@
 # Flink Docker Images
 
-This repository offers a set of Apache Flink Docker images maintained by LakeSail.
+This repository offers a set of Apache Flink Docker images maintained by [LakeSail](https://lakesail.com/).
 The LakeSail versions of the Flink Docker images are based on the official Flink Docker images and include additional features such as Python support.
 
-Use the following command to pull the image from GitHub Container Registry.
+Use the following command to pull the images from GitHub Container Registry.
 Replace `$TAG` with the desired tag.
 A list of available tags can be found in the [package homepage](https://github.com/orgs/lakehq/packages/container/package/flink).
 
 ```bash
 docker pull ghcr.io/lakehq/flink:$TAG
 ```
 
+More details about the images can be found in the [documentation](https://docs.lakesail.com/flink-docker-images/).
+
 ## GitHub Container Registry Packages
 
 * [lakehq/flink](https://github.com/orgs/lakehq/packages/container/package/flink)
@@ -21,9 +23,26 @@ docker pull ghcr.io/lakehq/flink:$TAG
 Use the following command(s) to build the images locally.
 
 ```bash
-docker build . -f docker/flink1.17/Dockerfile -t lakehq/flink:1.17.1-python3.10
+docker build . -t lakehq/flink:1.18.1-python3.10
 ```
 
+## References
+
+### Apache Flink Releases
+
+* [Apache Flink 1.17.0 Release Announcement](https://flink.apache.org/2023/03/23/announcing-the-release-of-apache-flink-1.17/)
+* [Apache Flink 1.17.1 Release Announcement](https://flink.apache.org/2023/05/25/apache-flink-1.17.1-release-announcement/)
+* [Apache Flink 1.17.2 Release Announcement](https://flink.apache.org/2023/11/29/apache-flink-1.17.2-release-announcement/)
+* [Apache Flink 1.18.0 Release Announcement](https://flink.apache.org/2023/10/24/announcing-the-release-of-apache-flink-1.18/)
+* [Apache Flink 1.18.1 Release Announcement](https://flink.apache.org/2024/01/19/apache-flink-1.18.1-release-announcement/)
+
+### Python Versions
+
+* [Status of Python Versions](https://devguide.python.org/versions/)
+* [Python 3.10 Release Schedule](https://peps.python.org/pep-0619/)
+* [Python 3.11 Release Schedule](https://peps.python.org/pep-0664/)
+* [Python 3.12 Release Schedule](https://peps.python.org/pep-0693/)
+
 ## License
 
 Licensed under the Apache License, Version 2.0: https://www.apache.org/licenses/LICENSE-2.0

bin/flink-fs-cp.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+FLINK_HOME="${FLINK_HOME:-/opt/flink}"
+FLINK_CONF_DIR="${FLINK_CONF_DIR:-${FLINK_HOME}/conf}"
+
+export FLINK_CONF_DIR
+
+FLINK_VERSION="$(basename "${FLINK_HOME}"/lib/flink-dist-*.jar)"
+FLINK_VERSION="${FLINK_VERSION#flink-dist-}"
+FLINK_VERSION="${FLINK_VERSION%.jar}"
+
+java -cp "${FLINK_HOME}/opt/flink-fs-utils-${FLINK_VERSION}.jar:${FLINK_HOME}/lib/flink-dist-${FLINK_VERSION}.jar" \
+    org.apache.flink.core.fs.CopyTool "$@"

pom.xml

@@ -8,9 +8,9 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>com.lakesail</groupId>
+    <groupId>org.apache.flink</groupId>
     <artifactId>flink-docker-images</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>1.18-SNAPSHOT</version>
 
     <packaging>pom</packaging>
 

tools/flink-fs-utils/pom.xml

@@ -4,9 +4,14 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>org.apache.flink</groupId>
+    <parent>
+        <groupId>org.apache.flink</groupId>
+        <artifactId>flink-docker-images</artifactId>
+        <version>1.18-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
     <artifactId>flink-fs-utils</artifactId>
-    <version>1.17.1</version>
 
     <properties>
         <flink.version>${project.version}</flink.version>