Use staged builds to minimize final image size

So that redundant things do not end in final image: - Git repo history - Test directories - Git tool and its deps And drop explicit installation of: - jemalloc & GLX: nothing uses them (in ChatQnA at least), and for testing it's trivial to create image adding those on top: https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#switch-memory-allocator - langchain_core: GenAIComps install langchain which already depends on that This demonstrates that only 2-3 lines in the Dockerfiles are unique, and everything before those can be removed with a common base image. Signed-off-by: Eero Tamminen <[email protected]>
opea-project · Nov 22, 2024 · 95e0c76 · 95e0c76
1 parent ac47042
commit 95e0c76
Show file tree

Hide file tree

Showing 18 changed files with 592 additions and 321 deletions.
diff --git a/AudioQnA/Dockerfile b/AudioQnA/Dockerfile
@@ -1,32 +1,48 @@
-
-
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-FROM python:3.11-slim
+# Stage 1: base setup used by other stages
+FROM python:3.11-slim AS base
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    git
+RUN apt-get update
+
+ENV HOME=/home/user
 
 RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
+    mkdir -p $HOME && \
+    chown -R user $HOME
 
-WORKDIR /home/user/
-RUN git clone https://github.com/opea-project/GenAIComps.git
+WORKDIR $HOME
 
-WORKDIR /home/user/GenAIComps
-RUN pip install --no-cache-dir --upgrade pip setuptools && \
-    pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt
 
-COPY ./audioqna.py /home/user/audioqna.py
+# Stage 2: latest GenAIComps sources
+FROM base AS git
+
+RUN apt-get install -y --no-install-recommends git
+
+RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git
+
 
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
+# Stage 3: common layer shared by services using GenAIComps
+FROM base AS comps-base
+
+# copy just relevant parts
+COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps
+COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/
+
+WORKDIR $HOME/GenAIComps
+RUN pip install --no-cache-dir --upgrade pip setuptools && \
+    pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt
+WORKDIR $HOME
+
+ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps
 
 USER user
 
-WORKDIR /home/user
+
+# Stage 4: unique part
+FROM comps-base
+
+COPY ./audioqna.py $HOME/audioqna.py
 
 ENTRYPOINT ["python", "audioqna.py"]
diff --git a/AudioQnA/Dockerfile.multilang b/AudioQnA/Dockerfile.multilang
@@ -1,32 +1,48 @@
-
-
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-FROM python:3.11-slim
+# Stage 1: base setup used by other stages
+FROM python:3.11-slim AS base
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    git
+RUN apt-get update
+
+ENV HOME=/home/user
 
 RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
+    mkdir -p $HOME && \
+    chown -R user $HOME
 
-WORKDIR /home/user/
-RUN git clone https://github.com/opea-project/GenAIComps.git
+WORKDIR $HOME
 
-WORKDIR /home/user/GenAIComps
-RUN pip install --no-cache-dir --upgrade pip setuptools && \
-    pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt
 
-COPY ./audioqna_multilang.py /home/user/audioqna_multilang.py
+# Stage 2: latest GenAIComps sources
+FROM base AS git
+
+RUN apt-get install -y --no-install-recommends git
+
+RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git
+
 
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
+# Stage 3: common layer shared by services using GenAIComps
+FROM base AS comps-base
+
+# copy just relevant parts
+COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps
+COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/
+
+WORKDIR $HOME/GenAIComps
+RUN pip install --no-cache-dir --upgrade pip setuptools && \
+    pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt
+WORKDIR $HOME
+
+ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps
 
 USER user
 
-WORKDIR /home/user
+
+# Stage 4: unique part
+FROM comps-base
+
+COPY ./audioqna_multilang.py $HOME/audioqna_multilang.py
 
 ENTRYPOINT ["python", "audioqna_multilang.py"]
diff --git a/AvatarChatbot/Dockerfile b/AvatarChatbot/Dockerfile
@@ -1,33 +1,48 @@
-
-
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-FROM python:3.11-slim
+# Stage 1: base setup used by other stages
+FROM python:3.11-slim AS base
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    vim \
-    git
+RUN apt-get update
+
+ENV HOME=/home/user
 
 RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
+    mkdir -p $HOME && \
+    chown -R user $HOME
 
-WORKDIR /home/user/
-RUN git clone https://github.com/opea-project/GenAIComps.git
-WORKDIR /home/user/GenAIComps
+WORKDIR $HOME
 
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt
 
-COPY ./avatarchatbot.py /home/user/avatarchatbot.py
+# Stage 2: latest GenAIComps sources
+FROM base AS git
+
+RUN apt-get install -y --no-install-recommends git
+
+RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git
+
 
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
+# Stage 3: common layer shared by services using GenAIComps
+FROM base AS comps-base
+
+# copy just relevant parts
+COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps
+COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/
+
+WORKDIR $HOME/GenAIComps
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt
+WORKDIR $HOME
+
+ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps
 
 USER user
 
-WORKDIR /home/user
+
+# Stage 4: unique part
+FROM comps-base
+
+COPY ./avatarchatbot.py $HOME/avatarchatbot.py
 
 ENTRYPOINT ["python", "avatarchatbot.py"]
diff --git a/ChatQnA/Dockerfile b/ChatQnA/Dockerfile
@@ -1,35 +1,48 @@
-
-
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-FROM python:3.11-slim
+# Stage 1: base setup used by other stages
+FROM python:3.11-slim AS base
+
+RUN apt-get update
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    git
+ENV HOME=/home/user
 
 RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
+    mkdir -p $HOME && \
+    chown -R user $HOME
 
-WORKDIR /home/user/
-RUN git clone https://github.com/opea-project/GenAIComps.git
+WORKDIR $HOME
 
-WORKDIR /home/user/GenAIComps
-RUN pip install --no-cache-dir --upgrade pip setuptools && \
-    pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt && \
-    pip install --no-cache-dir langchain_core
 
-COPY ./chatqna.py /home/user/chatqna.py
+# Stage 2: latest GenAIComps sources
+FROM base AS git
+
+RUN apt-get install -y --no-install-recommends git
+
+RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git
 
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
+
+# Stage 3: common layer shared by services using GenAIComps
+FROM base AS comps-base
+
+# copy just relevant parts
+COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps
+COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/
+
+WORKDIR $HOME/GenAIComps
+RUN pip install --no-cache-dir --upgrade pip setuptools && \
+    pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt
+WORKDIR $HOME
+
+ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps
 
 USER user
 
-WORKDIR /home/user
 
-RUN echo 'ulimit -S -n 999999' >> ~/.bashrc
+# Stage 4: unique part
+FROM comps-base
+
+COPY ./chatqna.py $HOME/chatqna.py
 
 ENTRYPOINT ["python", "chatqna.py"]
diff --git a/ChatQnA/Dockerfile.guardrails b/ChatQnA/Dockerfile.guardrails
@@ -1,35 +1,48 @@
-
-
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-FROM python:3.11-slim
+# Stage 1: base setup used by other stages
+FROM python:3.11-slim AS base
+
+RUN apt-get update
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
-    libgl1-mesa-glx \
-    libjemalloc-dev \
-    git
+ENV HOME=/home/user
 
 RUN useradd -m -s /bin/bash user && \
-    mkdir -p /home/user && \
-    chown -R user /home/user/
+    mkdir -p $HOME && \
+    chown -R user $HOME
 
-WORKDIR /home/user/
-RUN git clone https://github.com/opea-project/GenAIComps.git
+WORKDIR $HOME
 
-WORKDIR /home/user/GenAIComps
-RUN pip install --no-cache-dir --upgrade pip setuptools && \
-    pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt && \
-    pip install --no-cache-dir langchain_core
 
-COPY ./chatqna.py /home/user/chatqna.py
+# Stage 2: latest GenAIComps sources
+FROM base AS git
+
+RUN apt-get install -y --no-install-recommends git
+
+RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git
 
-ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
+
+# Stage 3: common layer shared by services using GenAIComps
+FROM base AS comps-base
+
+# copy just relevant parts
+COPY --from=git $HOME/GenAIComps/comps $HOME/GenAIComps/comps
+COPY --from=git $HOME/GenAIComps/*.* $HOME/GenAIComps/LICENSE $HOME/GenAIComps/
+
+WORKDIR $HOME/GenAIComps
+RUN pip install --no-cache-dir --upgrade pip setuptools && \
+    pip install --no-cache-dir -r $HOME/GenAIComps/requirements.txt
+WORKDIR $HOME
+
+ENV PYTHONPATH=$PYTHONPATH:$HOME/GenAIComps
 
 USER user
 
-WORKDIR /home/user
 
-RUN echo 'ulimit -S -n 999999' >> ~/.bashrc
+# Stage 4: unique part
+FROM comps-base
+
+COPY ./chatqna.py $HOME/chatqna.py
 
 ENTRYPOINT ["python", "chatqna.py", "--with-guardrails"]