From e558384d75e50fed95337380bc7959a29e5dcc52 Mon Sep 17 00:00:00 2001 From: Michal Franczel Date: Tue, 1 Sep 2026 14:56:18 +0200 Subject: [PATCH 1/3] feat(cd): publish merged full toolkit OCI bundle --- .github/actions/push-oci-artifact/action.yml | 34 +++++- .github/workflows/cd.yml | 121 ++++++++++++++++--- 2 files changed, 134 insertions(+), 21 deletions(-) diff --git a/.github/actions/push-oci-artifact/action.yml b/.github/actions/push-oci-artifact/action.yml index 7d4d94a..3af8c84 100644 --- a/.github/actions/push-oci-artifact/action.yml +++ b/.github/actions/push-oci-artifact/action.yml @@ -82,7 +82,13 @@ runs: ls -la "$(dirname "${file_path}")" >&2 || true exit 1 fi - cp "${file_path}" "${rootfs_dir}/$(basename "${file_path}")" + file_name="$(basename "${file_path}")" + staged_path="${rootfs_dir}/${file_name}" + if [ -e "${staged_path}" ] || [ -L "${staged_path}" ]; then + echo "Error: multiple OCI artifact files have the same basename: ${file_name}" >&2 + exit 1 + fi + ln -s "$(realpath "${file_path}")" "${staged_path}" file_count=$((file_count + 1)) done <<< "${FILES}" @@ -91,12 +97,30 @@ runs: exit 1 fi - layer_tar="${workdir}/rootfs.tar" layer_tar_gz="${workdir}/rootfs.tar.gz" - tar -C "${rootfs_dir}" -cf "${layer_tar}" . - pigz -n -6 -p "$(nproc)" -c "${layer_tar}" > "${layer_tar_gz}" + layer_diff_id_path="${workdir}/layer.diff-id" + layer_diff_id_pipe="${workdir}/layer.diff-id.pipe" + mkfifo "${layer_diff_id_pipe}" + + ( + sha256sum < "${layer_diff_id_pipe}" | awk '{print $1}' > "${layer_diff_id_path}" + ) & + layer_diff_id_pid=$! + + if ! tar -h -C "${rootfs_dir}" -cf - . \ + | tee "${layer_diff_id_pipe}" \ + | pigz -n -6 -p "$(nproc)" > "${layer_tar_gz}"; then + wait "${layer_diff_id_pid}" || true + echo "Error: failed to create the OCI rootfs layer" >&2 + exit 1 + fi + wait "${layer_diff_id_pid}" - layer_diff_id="$(sha256sum "${layer_tar}" | awk '{print $1}')" + layer_diff_id="$(< "${layer_diff_id_path}")" + if [[ ! "${layer_diff_id}" =~ ^[0-9a-f]{64}$ ]]; then + echo "Error: failed to calculate the OCI rootfs layer diff ID" >&2 + exit 1 + fi config_path="${workdir}/config.json" cat > "${config_path}" <> "$GITHUB_OUTPUT" + build-and-push-artifacts: name: Build and push artifacts for Python ${{ matrix.python_version }} runs-on: ubuntu-latest + needs: define-toolkit-python-versions # Only run for base repo, not forks or dependabot if: (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]' strategy: fail-fast: false matrix: - python_version: ["3.10", "3.11", "3.12", "3.13"] + python_version: ${{ fromJSON(needs.define-toolkit-python-versions.outputs.python_versions) }} permissions: id-token: write contents: read @@ -135,15 +149,6 @@ jobs: aws_role_arn: ${{ secrets.AWS_PRODUCTION_ROLE_ARN }} # OCI ARTIFACT UPLOADS - - name: Push toolkit bundle OCI artifact - uses: ./.github/actions/push-oci-artifact - with: - artifact_ref: docker.io/deepnote/toolkit-bundle:${{ steps.version.outputs.VERSION }}-python${{ matrix.python_version }} - artifact_type: application/vnd.deepnote.toolkit.bundle.v1 - files: dist/python${{ matrix.python_version }}.tar:application/vnd.deepnote.toolkit.python-bundle.v1.tar - toolkit_version: ${{ steps.version.outputs.VERSION }} - annotations: com.deepnote.toolkit.python-version=${{ matrix.python_version }} - - name: Push toolkit bundle OCI zstd artifact uses: ./.github/actions/push-oci-artifact with: @@ -165,6 +170,14 @@ jobs: files: dist/installer.zip:application/zip toolkit_version: ${{ steps.version.outputs.VERSION }} + - name: Upload toolkit bundle artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: toolkit-bundle-${{ steps.version.outputs.VERSION }}-python${{ matrix.python_version }} + path: dist/python${{ matrix.python_version }}.tar + if-no-files-found: error + retention-days: 1 + - name: Upload toolkit constraints artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: @@ -172,6 +185,81 @@ jobs: path: dist/constraints${{ matrix.python_version }}.txt if-no-files-found: error + push-toolkit-bundle-oci: + name: Push full toolkit bundle OCI artifact + runs-on: ubuntu-latest + needs: + - define-toolkit-python-versions + - build-and-push-artifacts + # Only run for base repo, not forks or dependabot + if: (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]' + permissions: + contents: read + env: + PYTHON_VERSIONS: ${{ needs.define-toolkit-python-versions.outputs.python_versions }} + steps: + - name: Checkout code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + with: + persist-credentials: false + + - name: Login to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3 + with: + username: deepnotebot + password: ${{ secrets.DOCKERHUB_PASS }} + + - name: Export version + id: version + uses: ./.github/actions/export-version + + - name: Download toolkit bundle artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: toolkit-bundle-${{ steps.version.outputs.VERSION }}-python* + path: dist/ + merge-multiple: true + + - name: Build full toolkit bundle OCI files input + id: bundle_oci_files + shell: bash + run: | + set -euo pipefail + + bundle_files="$(mktemp)" + mapfile -t python_versions < <(jq -r '.[]' <<< "${PYTHON_VERSIONS}") + if [ "${#python_versions[@]}" -eq 0 ]; then + echo "Error: no Python versions were configured" >&2 + exit 1 + fi + python_versions_csv="$(IFS=,; echo "${python_versions[*]}")" + + for python_version in "${python_versions[@]}"; do + bundle_file="dist/python${python_version}.tar" + if [ ! -f "${bundle_file}" ]; then + echo "Error: toolkit bundle was not downloaded: ${bundle_file}" >&2 + find dist -maxdepth 1 -type f -print >&2 + exit 1 + fi + printf '%s:application/vnd.deepnote.toolkit.python-bundle.v1.tar\n' "${bundle_file}" >> "${bundle_files}" + done + + { + echo "files<> "${GITHUB_OUTPUT}" + + - name: Push full toolkit bundle OCI artifact + uses: ./.github/actions/push-oci-artifact + with: + artifact_ref: docker.io/deepnote/toolkit-bundle:${{ steps.version.outputs.VERSION }}-full + artifact_type: application/vnd.deepnote.toolkit.bundle.v1 + files: ${{ steps.bundle_oci_files.outputs.files }} + toolkit_version: ${{ steps.version.outputs.VERSION }} + annotations: com.deepnote.toolkit.python-versions=${{ steps.bundle_oci_files.outputs.python_versions }} + push-toolkit-constraints-oci: name: Push toolkit constraints OCI artifact runs-on: ubuntu-latest @@ -238,6 +326,7 @@ jobs: runs-on: ubuntu-latest needs: - build-and-push-artifacts + - push-toolkit-bundle-oci - push-toolkit-constraints-oci # Only run if the artifact jobs ran (i.e., not for forks or dependabot) if: always() && (github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request') && github.actor != 'dependabot[bot]' @@ -245,18 +334,20 @@ jobs: - name: Check artifact job results env: BUILD_RESULT: ${{ needs.build-and-push-artifacts.result }} + BUNDLE_RESULT: ${{ needs.push-toolkit-bundle-oci.result }} CONSTRAINTS_RESULT: ${{ needs.push-toolkit-constraints-oci.result }} run: | build_result="${BUILD_RESULT}" + bundle_result="${BUNDLE_RESULT}" constraints_result="${CONSTRAINTS_RESULT}" - if [[ $build_result == "success" && $constraints_result == "success" ]]; then + if [[ $build_result == "success" && $bundle_result == "success" && $constraints_result == "success" ]]; then echo "All artifact jobs succeeded" exit 0 - elif [[ $build_result == "cancelled" || $constraints_result == "cancelled" ]]; then + elif [[ $build_result == "cancelled" || $bundle_result == "cancelled" || $constraints_result == "cancelled" ]]; then echo "One or more artifact jobs were cancelled" exit 1 else - echo "One or more artifact jobs failed: build=${build_result}, constraints=${constraints_result}" + echo "One or more artifact jobs failed: build=${build_result}, bundle=${bundle_result}, constraints=${constraints_result}" exit 1 fi @@ -533,8 +624,6 @@ jobs: runs-on: ubuntu-latest needs: build-and-push-artifacts-status if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') - env: - PYTHON_VERSIONS: "3.10,3.11,3.12,3.13" steps: - name: Checkout code uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 @@ -556,7 +645,7 @@ jobs: - name: Build toolkit-cache docker image env: - PYTHON_VERS: ${{ env.PYTHON_VERSIONS }} + PYTHON_VERS: ${{ join(fromJSON(env.TOOLKIT_PYTHON_VERSIONS), ',') }} run: | docker build \ --progress plain \ From 70513fec3da2a45a5634e06ec9d6e906afe33202 Mon Sep 17 00:00:00 2001 From: Michal Franczel Date: Tue, 1 Sep 2026 15:07:42 +0200 Subject: [PATCH 2/3] fix(cd): set OCI bundle artifact type --- .github/actions/push-oci-artifact/action.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/actions/push-oci-artifact/action.yml b/.github/actions/push-oci-artifact/action.yml index 3af8c84..f3f5135 100644 --- a/.github/actions/push-oci-artifact/action.yml +++ b/.github/actions/push-oci-artifact/action.yml @@ -142,6 +142,7 @@ runs: push "${ARTIFACT_REF}" --config "config.json:application/vnd.oci.image.config.v1+json" + --artifact-type "${ARTIFACT_TYPE}" ) for annotation in "${manifest_annotations[@]}"; do oras_args+=(--annotation "${annotation}") @@ -150,7 +151,19 @@ runs: oras "${oras_args[@]}" ) - oras manifest fetch "${ARTIFACT_REF}" >/dev/null + + published_manifest_path="${workdir}/published-manifest.json" + oras manifest fetch "${ARTIFACT_REF}" > "${published_manifest_path}" + if ! jq -e --arg artifact_type "${ARTIFACT_TYPE}" ' + .mediaType == "application/vnd.oci.image.manifest.v1+json" and + .artifactType == $artifact_type and + .config.mediaType == "application/vnd.oci.image.config.v1+json" and + (.layers | length) == 1 and + .layers[0].mediaType == "application/vnd.oci.image.layer.v1.tar+gzip" + ' "${published_manifest_path}" >/dev/null; then + echo "Error: published OCI image manifest has an unexpected shape or artifact type: ${ARTIFACT_TYPE}" >&2 + exit 1 + fi echo "Pushed ${ARTIFACT_REF}" - name: Push OCI direct file artifact From 09f99fa991fc571fa5b42f4aa0e6c323ddc223aa Mon Sep 17 00:00:00 2001 From: Michal Franczel Date: Tue, 1 Sep 2026 17:37:58 +0200 Subject: [PATCH 3/3] Revert "fix(cd): set OCI bundle artifact type" This reverts commit 70513fec3da2a45a5634e06ec9d6e906afe33202. --- .github/actions/push-oci-artifact/action.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/actions/push-oci-artifact/action.yml b/.github/actions/push-oci-artifact/action.yml index f3f5135..3af8c84 100644 --- a/.github/actions/push-oci-artifact/action.yml +++ b/.github/actions/push-oci-artifact/action.yml @@ -142,7 +142,6 @@ runs: push "${ARTIFACT_REF}" --config "config.json:application/vnd.oci.image.config.v1+json" - --artifact-type "${ARTIFACT_TYPE}" ) for annotation in "${manifest_annotations[@]}"; do oras_args+=(--annotation "${annotation}") @@ -151,19 +150,7 @@ runs: oras "${oras_args[@]}" ) - - published_manifest_path="${workdir}/published-manifest.json" - oras manifest fetch "${ARTIFACT_REF}" > "${published_manifest_path}" - if ! jq -e --arg artifact_type "${ARTIFACT_TYPE}" ' - .mediaType == "application/vnd.oci.image.manifest.v1+json" and - .artifactType == $artifact_type and - .config.mediaType == "application/vnd.oci.image.config.v1+json" and - (.layers | length) == 1 and - .layers[0].mediaType == "application/vnd.oci.image.layer.v1.tar+gzip" - ' "${published_manifest_path}" >/dev/null; then - echo "Error: published OCI image manifest has an unexpected shape or artifact type: ${ARTIFACT_TYPE}" >&2 - exit 1 - fi + oras manifest fetch "${ARTIFACT_REF}" >/dev/null echo "Pushed ${ARTIFACT_REF}" - name: Push OCI direct file artifact