SRE-2257 ci: no proxy option for docker

utils/docker/Dockerfile.el.8

@@ -22,6 +22,16 @@ ARG REPO_FILE_URL
 ARG JENKINS_URL
 ARG REPOS
 ARG DAOS_LAB_CA_FILE_URL
+
+# Accept DAOS_NO_PROXY at build time
+ARG DAOS_NO_PROXY
+# Propagate into the build environment
+ENV no_proxy=${DAOS_NO_PROXY}
+ENV NO_PROXY=${DAOS_NO_PROXY}
+# Persist into /etc/environment for use by shells and services
+RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \
+    echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment
+
 # script to install OS updates basic tools and daos dependencies
 COPY ./utils/scripts/install-el8.sh /tmp/install.sh
 # script to setup local repo if available
@@ -37,9 +47,7 @@ FROM basic
 # with a local repository, yet needing a proxy to reach outside repositories.
 # This needs to be moved to a shell script like above in the future to
 # properly only remove the proxy variables only when they need to be removed
-RUN if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "\
-        direct="${direct%%/*}"; export no_proxy="${direct}"; fi; \
-    dnf upgrade &&        \
+RUN dnf upgrade &&        \
     /tmp/install.sh &&    \
     dnf clean all &&      \
     rm -f /tmp/install.sh
@@ -59,8 +67,6 @@ RUN mkdir -p /opt/daos /mnt/daos /var/run/daos_server /var/run/daos_agent /home/
     chown -R daos_server.daos_server /opt/daos /mnt/daos /var/run/daos_server /home/daos &&  \
     chown daos_agent.daos_agent /var/run/daos_agent
 
-ARG JENKINS_URL
-
 USER daos_server:daos_server
 
 # Setup a python venv so that python packages can be installed locally.
@@ -96,8 +102,6 @@ ARG DAOS_PACKAGES_BUILD=yes
 # ensure that latest dependencies are used.
 USER root:root
 RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || {                                               \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "              \
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi;                   \
         dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \
         dnf clean all;                                                                 \
     }
@@ -106,8 +110,6 @@ USER daos_server:daos_server
 ARG DEPS_JOBS=1
 
 RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || {                            \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "\
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi; \
         scons --build-deps=only --jobs $DEPS_JOBS PREFIX=/opt/daos  \
               TARGET_TYPE=$DAOS_TARGET_TYPE &&                      \
         ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build *.gz); \
@@ -118,8 +120,7 @@ COPY --chown=daos_server:daos_server utils/sl utils/sl
 
 # Build third party RPMs
 RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "\
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi; \
+    export DISTRO="el8" &&                                                  \
 	utils/rpms/build_packages.sh deps &&                                    \
 	mkdir -p /home/daos/rpms &&                                             \
 	mv *.rpm /home/daos/rpms;                                               \
@@ -129,10 +130,7 @@ USER root:root
 # force an upgrade to get any newly built RPMs, but only if CB1 is set.
 ARG CB1
 RUN [ -z "$CB1" ] || {                                                     \
-    if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "      \
-        direct="${direct%%/*}"; export no_proxy="${direct}"; fi;           \
-    dnf upgrade                                                            \
-        --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \
+    dnf upgrade --exclude=spdk,spdk-devel,dpdk-devel,dpdk,mercury-devel,mercury && \
     dnf clean all;                                                         \
     }
 USER daos_server:daos_server
@@ -158,8 +156,6 @@ ARG DAOS_BUILD=$DAOS_DEPS_BUILD
 
 # Build DAOS
 RUN [ "$DAOS_BUILD" != "yes" ] || {                                        \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "  \
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi;      \
         scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER     \
               FIRMWARE_MGMT=1 BUILD_TYPE=$DAOS_BUILD_TYPE TARGET_TYPE=$DAOS_TARGET_TYPE && \
         ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build) &&           \
@@ -171,8 +167,6 @@ COPY --chown=daos_server:daos_server utils utils
 
 # Build DAOS RPMs
 RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "  \
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi;       \
 	utils/rpms/build_packages.sh daos &&                               \
 	mkdir -p /home/daos/rpms &&                                        \
 	cp *.rpm /home/daos/rpms;                                          \
@@ -188,8 +182,6 @@ WORKDIR /home/daos/daos/src/client/java
 ARG DAOS_JAVA_BUILD=$DAOS_BUILD
 
 RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || {                                                      \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "                     \
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi;                          \
         mkdir /home/daos/.m2 &&                                                               \
         cp /home/daos/daos/utils/scripts/helpers/maven-settings.xml.in /home/daos/.m2/settings.xml &&      \
         export JAVA_HOME=$(daos-java/find_java_home.sh) && \

utils/docker/Dockerfile.el.9

@@ -22,6 +22,16 @@ ARG REPO_FILE_URL
 ARG JENKINS_URL
 ARG REPOS
 ARG DAOS_LAB_CA_FILE_URL
+
+# Accept DAOS_NO_PROXY at build time
+ARG DAOS_NO_PROXY
+# Propagate into the build environment
+ENV no_proxy=${DAOS_NO_PROXY}
+ENV NO_PROXY=${DAOS_NO_PROXY}
+# Persist into /etc/environment for use by shells and services
+RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \
+    echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment
+
 # script to install OS updates basic tools and daos dependencies
 COPY ./utils/scripts/install-el9.sh /tmp/install.sh
 # script to setup local repo if available

utils/docker/Dockerfile.leap.15

@@ -22,6 +22,16 @@ ARG REPO_FILE_URL
 ARG JENKINS_URL
 ARG REPOS
 ARG DAOS_LAB_CA_FILE_URL
+
+# Accept DAOS_NO_PROXY at build time
+ARG DAOS_NO_PROXY
+# Propagate into the build environment
+ENV no_proxy=${DAOS_NO_PROXY}
+ENV NO_PROXY=${DAOS_NO_PROXY}
+# Persist into /etc/environment for use by shells and services
+RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \
+    echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment
+
 # script to install OS updates basic tools and daos dependencies
 COPY ./utils/scripts/install-leap15.sh /tmp/install.sh
 # script to setup local repo if available
@@ -38,9 +48,7 @@ FROM basic
 # with a local repository, yet needing a proxy to reach outside repositories.
 # This needs to be moved to a shell script like above in the future to
 # properly only remove the proxy variables only when they need to be removed
-RUN if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "\
-        direct="${direct%%/*}"; export no_proxy="${direct}"; fi; \
-    dnf upgrade &&        \
+RUN dnf upgrade &&        \
     /tmp/install.sh &&    \
     dnf clean all &&      \
     rm -f /tmp/install.sh
@@ -96,8 +104,6 @@ ARG DAOS_PACKAGES_BUILD=yes
 # The dnf upgrade can add or re-enable distro repositories.
 USER root:root
 RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || {                                                             \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "                            \
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi;                                 \
         dnf upgrade --exclude=fuse,fuse-libs,fuse-devel,libraft0,raft-devel,mercury,mercury-devel && \
         dnf clean all;                                                                               \
     }
@@ -106,8 +112,6 @@ USER daos_server:daos_server
 ARG DEPS_JOBS=1
 
 RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || {                            \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "\
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi; \
         scons --build-deps=only --jobs $DEPS_JOBS PREFIX=/opt/daos  \
               TARGET_TYPE=$DAOS_TARGET_TYPE &&                      \
         ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build *.gz); \
@@ -140,17 +144,13 @@ RUN if [ "$COMPILER" = "icc" ]; then
             rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && \
             dnf config-manager --add-repo https://yum.repos.intel.com/oneapi oneAPI;                     \
         fi;                                                                                              \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "\
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi; \
         dnf install intel-oneapi-compiler-dpcpp-cpp &&                                                   \
         dnf clean all;                                                                                   \
     fi
 
 # force an upgrade to get any newly built RPMs, but only if CB1 is set.
 ARG CB1
 RUN [ -z "$CB1" ] || {                                                                               \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "                            \
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi;                                 \
         dnf upgrade --exclude=fuse,fuse-libs,fuse-devel,libraft0,raft-devel,mercury,mercury-devel && \
         dnf clean all;                                                                               \
     }
@@ -175,8 +175,6 @@ ARG DAOS_BUILD=$DAOS_DEPS_BUILD
 
 # Build DAOS
 RUN [ "$DAOS_BUILD" != "yes" ] || {                                        \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "  \
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi;       \
         scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER     \
               BUILD_TYPE=$DAOS_BUILD_TYPE TARGET_TYPE=$DAOS_TARGET_TYPE    \
 	      FIRMWARE_MGMT=1 &&                                           \
@@ -205,8 +203,6 @@ WORKDIR /home/daos/daos/src/client/java
 ARG DAOS_JAVA_BUILD=$DAOS_BUILD
 
 RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || {                                                      \
-        if [ -n "$REPO_FILE_URL" ]; then direct="${REPO_FILE_URL##*//}; "                     \
-            direct="${direct%%/*}"; export no_proxy="${direct}"; fi;                          \
         mkdir /home/daos/.m2 &&                                                               \
         cp /home/daos/daos/utils/scripts/helpers/maven-settings.xml.in /home/daos/.m2/settings.xml &&      \
         mvn clean install -ntp -T 1C -DskipITs -Dgpg.skip -Ddaos.install.path=/opt/daos;      \

utils/docker/Dockerfile.ubuntu

@@ -22,6 +22,16 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # Install basic tools
 ARG DAOS_LAB_CA_FILE_URL
+
+# Accept DAOS_NO_PROXY at build time
+ARG DAOS_NO_PROXY
+# Propagate into the build environment
+ENV no_proxy=${DAOS_NO_PROXY}
+ENV NO_PROXY=${DAOS_NO_PROXY}
+# Persist into /etc/environment for use by shells and services
+RUN echo "no_proxy=${DAOS_NO_PROXY}" >> /etc/environment && \
+    echo "NO_PROXY=${DAOS_NO_PROXY}" >> /etc/environment
+
 # script to setup local repo if available and install packages
 COPY ./utils/scripts/helpers/repo-helper-ubuntu.sh /tmp/repo-helper.sh
 COPY ./utils/scripts/install-ubuntu.sh /tmp/install.sh

utils/scripts/helpers/repo-helper-el8.sh

@@ -74,8 +74,8 @@ if [ -n "$REPO_FILE_URL" ]; then
     popd
     # These may have been created in the Dockerfile must be removed
     # when using a local repository.
-    unset HTTPS_PROXY
-    unset https_proxy
+    # unset HTTPS_PROXY
+    # unset https_proxy
 fi
 dnf -y --disablerepo \*epel\* install dnf-plugins-core
 dnf -y config-manager --save --setopt=assumeyes=True
@@ -130,10 +130,9 @@ disable_repos /etc/yum.repos.d/ "${save_repos[@]}"
 if [ -n "$REPO_FILE_URL" ]; then
     trusted_host="${REPO_FILE_URL##*//}"
     trusted_host="${trusted_host%%/*}"; \
-    {
-        echo "[global]"
-        echo "trusted-host = ${trusted_host}"
-        echo "index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple"
-        echo "proxy = \"\""
-    } > /etc/pip.conf
+    cat <<EOF > /etc/pip.conf
+[global]
+    trusted-host = ${trusted_host}
+    index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple
+EOF
 fi

utils/scripts/helpers/repo-helper-el9.sh

@@ -126,10 +126,9 @@ disable_repos /etc/yum.repos.d/ "${save_repos[@]}"
 if [ -n "$REPO_FILE_URL" ]; then
     trusted_host="${REPO_FILE_URL##*//}"
     trusted_host="${trusted_host%%/*}"; \
-    {
-        echo "[global]"
-        echo "trusted-host = ${trusted_host}"
-        echo "index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple"
-        echo "proxy = "
-    } > /etc/pip.conf
+    cat <<EOF > /etc/pip.conf
+[global]
+    trusted-host = ${trusted_host}
+    index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple
+EOF
 fi

utils/scripts/helpers/repo-helper-leap15.sh

@@ -174,10 +174,9 @@ update-ca-certificates
 if [ -n "$REPO_FILE_URL" ]; then
     trusted_host="${REPO_FILE_URL##*//}"
     trusted_host="${trusted_host%%/*}"; \
-    {
-        echo "[global]"
-        echo "trusted-host = ${trusted_host}"
-        echo "index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple"
-        echo "proxy = \"\""
-     } > /etc/pip.conf
+    cat <<EOF > /etc/pip.conf
+[global]
+    trusted-host = ${trusted_host}
+    index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple
+EOF
 fi

utils/scripts/helpers/repo-helper-ubuntu.sh

@@ -111,10 +111,9 @@ apt-get clean all
 if [ -n "$REPO_FILE_URL" ]; then
     trusted_host="${REPO_FILE_URL##*//}"
     trusted_host="${trusted_host%%/*}"; \
-    {
-        echo "[global]"
-        echo "trusted-host = ${trusted_host}"
-        echo "index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple"
-        echo "proxy = \"\""
-    } > /etc/pip.conf
+    cat <<EOF > /etc/pip.conf
+[global]
+    trusted-host = ${trusted_host}
+    index-url = https://${trusted_host}/artifactory/api/pypi/pypi-proxy/simple
+EOF
 fi