common.sh

#!/usr/bin/env bash
#
# Copyright 2018, 2021 Delphix
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

export _RET
export _RET_LIST
export _RET_MIRROR_MAIN
export _RET_MIRROR_SECONDARY
export _SECRET_BUILD_ARGS
export DEBIAN_FRONTEND=noninteractive

export SUPPORTED_KERNEL_FLAVORS="generic aws gcp azure oracle"

#
# Used when fetching artifacts for external dependencies. Can be overridden
# for testing purposes to use jenkins-ops.<developer> instead.
#
export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}"

#
# The suite package builds run against, which may be different than the
# branch's default (see lib/container.sh).
#
export UBUNTU_DISTRIBUTION="${UBUNTU_DISTRIBUTION:-noble}"

#
# Whether generate_sbom() post-processes each CycloneDX SBOM before it is
# validated and published (see sanitize_sbom() and resources/sanitize-sbom.jq).
#
# Set to "false" to publish the raw Syft output instead, which keeps
# syft:metadata:virtualPath and so records where inside the package each
# component was found. That is of no use to a customer, but is how we determine
# where an unexpected component came from.
#
# Only the exact string "false" disables it, so that a typo leaves us with a
# publishable document rather than one recording our internal paths.
#
export CYCLONEDX_FILTERING="${CYCLONEDX_FILTERING:-true}"

source "$(dirname "${BASH_SOURCE[0]}")/container.sh"

#
# We currently support getting the linux kernel from 3 different sources:
#  1. "delphix": building it from code
#  2. "archive": dowloading from apt
#  3. "prebuilt": pre-built kernel stored in artifactory
#
export DEFAULT_LINUX_KERNEL_PACKAGE_SOURCE="delphix"

# shellcheck disable=SC2086
function enable_colors() {
	[[ -t 1 ]] && flags="" || flags="-T xterm"
	FMT_RED="$(tput $flags setaf 1)"
	FMT_GREEN="$(tput $flags setaf 2)"
	FMT_BOLD="$(tput $flags bold)"
	FMT_NF="$(tput $flags sgr0)"
	COLORS_ENABLED=true
}

function disable_colors() {
	FMT_RED=""
	FMT_GREEN=""
	FMT_BOLD=""
	FMT_NF=""
	COLORS_ENABLED=false
}

if [[ -t 1 ]] || [[ "$FORCE_COLORS" == "true" ]]; then
	enable_colors
else
	disable_colors
fi

function without_colors() {
	if [[ "$COLORS_ENABLED" == "true" ]]; then
		disable_colors
		"$@"
		enable_colors
	else
		"$@"
	fi
}

function echo_error() {
	echo -e "${FMT_BOLD}${FMT_RED}Error: $*${FMT_NF}" >&2
}

function echo_success() {
	echo -e "${FMT_BOLD}${FMT_GREEN}Success: $*${FMT_NF}"
}

function echo_bold() {
	echo -e "${FMT_BOLD}$*${FMT_NF}"
}

function die() {
	[[ $# -gt 0 ]] && echo_error "$*"
	exit 1
}

function logmust() {
	[[ "$LOGGING" == "false" ]] || echo Running: "$@" >&2
	"$@" || die "failed command '$*'"
}

#
# Package builds install build dependencies and kernel headers into the root
# filesystem they run on, so they must run inside the build container, whose
# root is disposable and whose codename matches the suite being built. The
# codename comparison below therefore holds by construction, and exists to catch
# a mismatched or stale image rather than to guard the caller's machine.
#
# On the LINUX_PKG_NO_CONTAINER escape hatch there is no container, and the same
# comparison becomes the only thing verifying that this host runs the suite whose
# apt sources setup.sh is about to point it at, and whose packages it is about to
# install. The two cases need different advice when they fail, so the failure
# below distinguishes them.
#
function check_running_system() {
	if [[ "$DISABLE_SYSTEM_CHECK" == "true" ]]; then
		echo "WARNING: System check disabled."
		return 0
	fi

	if ! running_in_container && [[ "$LINUX_PKG_NO_CONTAINER" != "true" ]]; then
		die "Expected to be running inside the build container." \
			"This is a bug in the re-exec logic in lib/container.sh."
	fi

	if ! (command -v lsb_release >/dev/null &&
		[[ $(lsb_release -cs) == "$UBUNTU_DISTRIBUTION" ]]); then
		echo_error "Running on ubuntu-$(lsb_release -cs 2>/dev/null || echo unknown)" \
			"but UBUNTU_DISTRIBUTION is '$UBUNTU_DISTRIBUTION'."
		if running_in_container; then
			die "The build container's rootfs does not match the suite being" \
				"built. Rebuild the image with LINUX_PKG_REBUILD_IMAGE=true."
		else
			die "LINUX_PKG_NO_CONTAINER is set, so this runs directly on this" \
				"host, which must itself run the suite being built. Either" \
				"unset LINUX_PKG_NO_CONTAINER and let the build run in a" \
				"container bootstrapped for $UBUNTU_DISTRIBUTION, or set" \
				"UBUNTU_DISTRIBUTION to this host's own suite."
		fi
	fi
}

#
# We need to have run setup.sh before running most linux-pkg commands.
# This checks if setup has been run before. Note that if the system was
# rebooted we want to rerun setup as cloud-init will reset apt sources on
# boot.
#
function run_setup_if_needed() {
	[[ -f /run/linux-pkg-setup ]] && return

	check_env TOP
	echo_bold "------------------------------------------------------------"
	echo_bold "Running setup script"
	echo_bold "------------------------------------------------------------"
	logmust "$TOP/setup.sh"
	echo_bold "------------------------------------------------------------"
}

#
# Resolve the URLs of the primary (Ubuntu archive) and secondary (PPA) package
# mirrors for the current branch, without modifying the system. If the URLs were
# passed in via the environment they are used as-is; otherwise the latest mirror
# snapshot for the branch is looked up.
#
# Sets _RET_MIRROR_MAIN and _RET_MIRROR_SECONDARY.
#
function resolve_mirror_urls() {
	check_env DEFAULT_GIT_BRANCH DELPHIX_RELEASE_VERSION
	local package_mirror_url latest_url
	local primary_url="$DELPHIX_PACKAGE_MIRROR_MAIN"
	local secondary_url="$DELPHIX_PACKAGE_MIRROR_SECONDARY"

	if [[ -z "$primary_url" ]] || [[ -z "$secondary_url" ]]; then
		#
		# Every product branch is served by the v2 (aptly-based)
		# mirror. The older linux-package-mirror.delphix.com name is
		# deprecated and is now just an alias for the same host.
		#
		latest_url="http://linux-package-mirror-v2.delphix.com/"

		if is_release_branch; then
			package_mirror_url="${latest_url}releases/${DELPHIX_RELEASE_VERSION}"
		else
			latest_url+="${DEFAULT_GIT_BRANCH}/latest/"
			package_mirror_url=$(curl -LfSs -o /dev/null -w '%{url_effective}' \
				"$latest_url" || die "Could not curl $latest_url")
			package_mirror_url="${package_mirror_url%/}"
		fi
		[[ -z "$primary_url" ]] && primary_url="${package_mirror_url}/ubuntu"
		[[ -z "$secondary_url" ]] && secondary_url="${package_mirror_url}/ppas"
	fi

	_RET_MIRROR_MAIN="$primary_url"
	_RET_MIRROR_SECONDARY="$secondary_url"
}

function is_release_branch() {
	check_env DEFAULT_GIT_BRANCH
	[[ "$DEFAULT_GIT_BRANCH" == release/* ]]
}

function check_package_exists() {
	local pkg="$1"

	check_env TOP
	echo "$pkg" | grep -q '/' && die "Package name should not contain '/'"
	[[ -d "$TOP/packages/$pkg" ]] || die "Unknown package '$pkg'."
}

function check_env() {
	local var val required

	required=true
	for var in "$@"; do
		if [[ "$var" == "--" ]]; then
			required=false
			continue
		fi

		val="${!var}"
		if $required && [[ -z "$val" ]]; then
			die "check_env: $var must be non-empty"
		fi
	done
}

function check_git_ref() {
	local ref
	for ref in "$@"; do
		if ! git show-ref -q "$ref"; then
			die "git ref '$ref' not found"
		fi
	done
}

#
# execute a hook from a package's config.sh
#
function stage() {
	typeset hook=$1
	shift 1

	check_env PACKAGE
	local stage_start=$SECONDS

	echo ""
	if type -t "$hook" >/dev/null; then
		echo_bold "PACKAGE $PACKAGE: STAGE $hook STARTED"
		logmust "$hook" "$@"
		echo_bold "PACKAGE $PACKAGE: STAGE $hook COMPLETED in" \
			"$((SECONDS - stage_start)) seconds"
	else
		echo_bold "PACKAGE $PACKAGE: SKIPPING UNDEFINED STAGE $hook"
	fi
	echo ""
}

function reset_package_config_variables() {
	local hook
	local var

	local hooks="
	prepare
	fetch
	build
	checkstyle
	update_upstream
	merge_with_upstream
	"

	for hook in $hooks; do
		unset "$hook"
	done

	local vars="
	PACKAGE_GIT_URL
	PACKAGE_GIT_BRANCH
	PACKAGE_VERSION
	PACKAGE_REVISION
	DEFAULT_PACKAGE_GIT_URL
	PACKAGE_DEPENDENCIES
	UPSTREAM_SOURCE_PACKAGE
	UPSTREAM_GIT_URL
	UPSTREAM_GIT_BRANCH
	WORKDIR
	PKGDIR
	PACKAGE_PREFIX
	FORCE_PUSH_ON_UPDATE
	SKIP_COPYRIGHTS_CHECK
	PACKAGE_NEEDS_DOCKER
	"

	for var in $vars; do
		unset "$var"
	done
}

function get_package_prefix() {
	local pkg="$1"
	local pkg_prefix

	#
	# We allow overriding package-specific configuration through
	# environment variables starting with <pkg_prefix> in
	# get_package_config_from_env(). We make sure that the names of new
	# packages can be converted to a valid <pkg_prefix>.
	#
	pkg_prefix="$(echo "$pkg" | tr - _ | tr '[:lower:]' '[:upper:]')"
	[[ "$pkg_prefix" =~ ^[A-Z][A-Z0-9_]*$ ]] ||
		die "Failed to convert package name '$pkg' to valid" \
			"prefix ($pkg_prefix)"
	_RET="$pkg_prefix"
}

#
# Loads configuration for building package passed in $1. High level tasks are:
#  1. Reset/Cleanup package configuration environment
#  2. Source default config for all packages: default-package-config.sh
#  3. Source default config for specific package: packages/PACKAGE/config.sh
#  4. Look at environment variables that can override default configs.
#  5. Validate config
#
function load_package_config() {
	export PACKAGE="$1"

	logmust check_package_exists "$PACKAGE"

	#
	# unset hooks and variables that are reserved for a package's config.
	#
	logmust reset_package_config_variables

	check_env TOP
	export PKGDIR="$TOP/packages/$PACKAGE"
	export WORKDIR="$PKGDIR/tmp"

	logmust source "$TOP/default-package-config.sh"
	logmust source "$PKGDIR/config.sh"

	#
	# A package's config.sh file can define default values for:
	#   GIT_URL, GIT_BRANCH, VERSION, REVISION.
	#
	# Those defaults can be overriden either by package-specific
	# environment variables or by parameters passed from command line.
	#
	logmust get_package_prefix "$PACKAGE"
	export PACKAGE_PREFIX="$_RET"
	logmust get_package_config_from_env

	#
	# Check that package configuration is valid
	#

	[[ -n "$DEFAULT_PACKAGE_GIT_URL" ]] ||
		die "$PACKAGE: DEFAULT_PACKAGE_GIT_URL is not defined. Set " \
			"it to 'none' if the source is not fetched from git"

	[[ "$DEFAULT_PACKAGE_GIT_URL" == https://* ]] ||
		[[ "$DEFAULT_PACKAGE_GIT_URL" == "none" ]] ||
		die "$PACKAGE: DEFAULT_PACKAGE_GIT_URL must begin with " \
			"https:// or be set to 'none'"

	local dependency
	local deps_array=()
	for dependency in $PACKAGE_DEPENDENCIES; do
		#
		# Check for special value @linux-kernel which resolves to
		# all flavors of linux kernel packages.
		#
		if [[ $dependency == '@linux-kernel' ]]; then
			logmust list_linux_kernel_packages
			deps_array+=("${_RET_LIST[@]}")
			continue
		fi
		(check_package_exists "$dependency") ||
			die "Invalid package dependency '$dependency'"
		deps_array+=("$dependency")
	done
	PACKAGE_DEPENDENCIES="${deps_array[*]}"

	#
	# Check for variables related to update_upstream() hook
	#
	local found=false
	if [[ -n "$UPSTREAM_GIT_URL" ]]; then
		[[ -n "$UPSTREAM_GIT_BRANCH" ]] ||
			die "$PACKAGE: UPSTREAM_GIT_BRANCH must also be" \
				"defined when UPSTREAM_GIT_URL is defined."
		found=true
	elif [[ -n "$UPSTREAM_GIT_BRANCH" ]]; then
		die "$PACKAGE: UPSTREAM_GIT_URL must also be defined when" \
			"UPSTREAM_GIT_BRANCH is defined."
	fi
	if [[ -n "$UPSTREAM_SOURCE_PACKAGE" ]]; then
		$found && die "$PACKAGE: UPSTREAM_SOURCE_PACKAGE and" \
			"UPSTREAM_GIT_URL are mutually exclusive."
		found=true
	fi
	if $found && ! type -t update_upstream >/dev/null; then
		die "$PACKAGE: update_upstream() hook must be defined when" \
			"either UPSTREAM_SOURCE_PACKAGE or UPSTREAM_GIT_URL" \
			"is set."
	fi

	#
	# Check that mandatory hooks are defined
	#
	for hook in fetch build; do
		type -t "$hook" >/dev/null ||
			die "$PACKAGE: Hook '$hook' missing."
	done
}

#
# Use different config sources to determine the values for:
#   PACKAGE_GIT_URL, PACKAGE_GIT_BRANCH, PACKAGE_REVISION
#
# The sources for the config, in decreasing order of priority, are:
#   1. Command line parameters passed to build script.
#   2. Package-specific environment variables {PACKAGE_PREFIX}_{SUFFIX}.
#      PACKAGE_PREFIX is the package's name in CAPS with '-' replaced by '_'.
#      E.g. CLOUD_INIT_GIT_URL sets PACKAGE_GIT_URL for package cloud-init.
#   3. DEFAULT_PACKAGE_{SUFFIX} variable defined in package's config.sh.
#      Only GIT_URL has this tier (DEFAULT_PACKAGE_GIT_URL); GIT_BRANCH and
#      REVISION fall straight from tier 2 to tier 4 below.
#   4. Global defaults for all packages, DEFAULT_{SUFFIX} (none for GIT_URL).
#
# This function should be called after loading a package's config.sh.
#
# A change to this precedence, or to buildpkg.sh's -g/-b/-r flags, needs
# docs/testing.md updated.
#
function get_package_config_from_env() {
	local var
	check_env PACKAGE_PREFIX

	echo "get_package_config_from_env(): using prefix: ${PACKAGE_PREFIX}_"

	var="${PACKAGE_PREFIX}_GIT_URL"
	if [[ -n "$PARAM_PACKAGE_GIT_URL" ]]; then
		PACKAGE_GIT_URL="$PARAM_PACKAGE_GIT_URL"
		echo "PARAM_PACKAGE_GIT_URL passed from '-g'"
	elif [[ -n "${!var}" ]]; then
		PACKAGE_GIT_URL="${!var}"
		echo "PACKAGE_GIT_URL set to value of ${var}"
	elif [[ -n "$DEFAULT_PACKAGE_GIT_URL" ]]; then
		PACKAGE_GIT_URL="$DEFAULT_PACKAGE_GIT_URL"
		echo "PACKAGE_GIT_URL set to value of DEFAULT_PACKAGE_GIT_URL"
	fi

	var="${PACKAGE_PREFIX}_GIT_BRANCH"
	if [[ -n "$PARAM_PACKAGE_GIT_BRANCH" ]]; then
		PACKAGE_GIT_BRANCH="$PARAM_PACKAGE_GIT_BRANCH"
		echo "PARAM_PACKAGE_GIT_BRANCH passed from '-b'"
	elif [[ -n "${!var}" ]]; then
		PACKAGE_GIT_BRANCH="${!var}"
		echo "PACKAGE_GIT_BRANCH set to value of ${var}"
	fi

	if [[ -z "$PACKAGE_GIT_BRANCH" ]]; then
		PACKAGE_GIT_BRANCH="$DEFAULT_GIT_BRANCH"
		echo "PACKAGE_GIT_BRANCH set to value of DEFAULT_GIT_BRANCH"
	fi

	var="${PACKAGE_PREFIX}_REVISION"
	if [[ -n "$PARAM_PACKAGE_REVISION" ]]; then
		PACKAGE_REVISION="$PARAM_PACKAGE_REVISION"
		echo "PACKAGE_REVISION passed from '-r'"
	elif [[ -n "${!var}" ]]; then
		PACKAGE_REVISION="${!var}"
		echo "PACKAGE_REVISION set to value of ${var}"
	fi

	if [[ -z "$PACKAGE_REVISION" ]]; then
		PACKAGE_REVISION="$DEFAULT_REVISION"
		echo "PACKAGE_REVISION set to value of DEFAULT_REVISION"
	fi

	export PACKAGE_GIT_URL
	export PACKAGE_GIT_BRANCH
	export PACKAGE_REVISION

	echo_bold "------------------------------------------------------------"
	echo_bold "PACKAGE_GIT_URL:      $PACKAGE_GIT_URL"
	echo_bold "PACKAGE_GIT_BRANCH:   $PACKAGE_GIT_BRANCH"
	echo_bold "PACKAGE_REVISION:     $PACKAGE_REVISION"
	echo_bold "------------------------------------------------------------"
}

function create_workdir() {
	check_env WORKDIR
	logmust sudo rm -rf "$WORKDIR"
	logmust mkdir "$WORKDIR"
	logmust rm -f "$TOP/workdir"
	logmust ln -s "$WORKDIR" "$TOP/workdir"
}

#
# apt install packages.
#
function install_pkgs() {
	for attempt in {1..3}; do
		echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
			"apt-get install -y --allow-downgrades $*"

		sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
			-y --allow-downgrades "$@" && return

		echo "apt-get install failed, retrying."
		sleep 10
	done
	die "apt-get install failed after $attempt attempts"
}

#
# Install build dependencies listed in the debian/control file of the package.
#
function install_build_deps_from_control_file() {
	logmust pushd "$WORKDIR/repo"
	logmust sudo env DEBIAN_FRONTEND=noninteractive mk-build-deps --install \
		--tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' \
		debian/control
	logmust popd
}

#
# Returns a list of all known packages in _RET_LIST.
#
function list_all_packages() {
	local pkg

	_RET_LIST=()

	for pkg in "$TOP/packages/"*; do
		pkg=$(basename "$pkg")
		if [[ -f "$TOP/packages/$pkg/config.sh" ]]; then
			_RET_LIST+=("$pkg")
		fi
	done
}

#
# Read a new-line separated list from a file, removing extra whitespace
# and comments. Return items in _RET_LIST.
#
function read_list() {
	local file="$1"

	local item
	local line

	_RET_LIST=()

	[[ -f "$file" ]] || die "Not a file: $file"

	while read -r line; do
		# trim whitespace
		item=$(echo "$line" | tr -d '[:space:]')
		[[ -z "$item" ]] && continue
		# ignore comments
		[[ ${item:0:1} == "#" ]] && continue
		_RET_LIST+=("$item")
	done <"$file" || die "Failed to read list: $file"
}

#
# Read a package-list file and return listed packages in _RET_LIST.
#
function read_package_list() {
	local file="$1"

	local pkg

	logmust read_list "$file"
	for pkg in "${_RET_LIST[@]}"; do
		check_package_exists "$pkg"
	done
}

#
# List all target kernel packages. By default, it returns all the kernel
# flavors supported and built by linux-pkg, however this can be overridden
# via TARGET_KERNEL_FLAVORS, which can be useful when testing changes against
# a specific, or even mainline kernel.
#
function list_linux_kernel_packages() {
	local kernel

	_RET_LIST=()
	if [[ -n "$TARGET_KERNEL_FLAVORS" ]]; then
		for kernel in $TARGET_KERNEL_FLAVORS; do
			(check_package_exists "linux-kernel-$kernel") ||
				die "Invalid entry '$kernel' in TARGET_KERNEL_FLAVORS"
			_RET_LIST+=("linux-kernel-$kernel")
		done
	else
		for kernel in $SUPPORTED_KERNEL_FLAVORS; do
			check_package_exists "linux-kernel-$kernel"
			_RET_LIST+=("linux-kernel-$kernel")
		done
	fi
	return 0
}

function install_shfmt() {
	if [[ ! -f /usr/local/bin/shfmt ]]; then
		local arch
		arch=$(dpkg-architecture -q DEB_HOST_ARCH)

		# The release names for shfmt don't use the actual
		# architecture strings, unfortunately.
		if [[ "$arch" == "arm64" ]]; then
			arch="arm"
		fi
		logmust sudo wget -nv -O /usr/local/bin/shfmt \
			"https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_$arch"
		logmust sudo chmod +x /usr/local/bin/shfmt
	fi
	echo "shfmt version $(shfmt -version) is installed."
}

#
# Install the tooling generate_sbom() needs. That is a default hook, shared
# unmodified by every package that sets SBOM_DEEP_SCAN, so the tools it runs
# belong with the rest of the generic build tooling installed here rather than
# being declared as a dependency by each of those packages individually.
#
# Unlike everything else installed from setup.sh these are Delphix-built
# packages with no apt source, so they are fetched from the same S3 location
# fetch_dependencies() pulls a package's dependencies from, then installed by
# path. apt rather than dpkg, because delphix-cyclonedx-cli has real
# dependencies (libicu and the usual shared libraries) that dpkg will not
# resolve.
#
# Best-effort by design: this runs before *every* package build, including the
# builds of syft and cyclonedx-cli themselves, and on a branch where neither
# has been published yet there is nothing to fetch. Failing hard would break
# every build on such a branch rather than just SBOM generation, so a missing
# artifact warns and moves on; generate_sbom() checks for the tools itself and
# fails loudly, for the only builds that actually need them.
#
function install_sbom_tools() {
	local pkg s3url tmpdir
	local debs=()

	tmpdir="$(mktemp -d)" || die "Failed to create a temporary directory"

	for pkg in syft cyclonedx-cli; do
		#
		# Run in a command substitution so that a failure to resolve
		# the URL (get_package_dependency_s3_url dies when a package
		# has no published artifacts) leaves $s3url empty here instead
		# of aborting setup.
		#
		s3url="$(
			get_package_dependency_s3_url "$pkg" >/dev/null 2>&1
			echo "$_RET"
		)"
		if [[ -z "$s3url" ]]; then
			echo "WARNING: no published artifacts found for '$pkg';" \
				"skipping the SBOM tooling install. Builds of" \
				"packages that set SBOM_DEEP_SCAN will fail until" \
				"'$pkg' has been built for this branch."
			logmust rm -rf "$tmpdir"
			return 0
		fi

		[[ "$s3url" != */ ]] && s3url="$s3url/"
		logmust mkdir -p "$tmpdir/$pkg"
		logmust aws s3 cp --only-show-errors --recursive \
			"$s3url" "$tmpdir/$pkg/"
	done

	debs=("$tmpdir"/*/*.deb)
	[[ -e "${debs[0]}" ]] ||
		die "No .deb found in the fetched syft/cyclonedx-cli artifacts"

	logmust install_pkgs "${debs[@]}"
	logmust rm -rf "$tmpdir"
}

#
# Install kernel headers packages for all target kernels.
# The kernel packages are fetched from S3.
#
function install_kernel_headers() {
	logmust determine_target_kernels
	check_env KERNEL_VERSIONS DEPDIR

	logmust list_linux_kernel_packages
	# Note: linux packages returned in _RET_LIST

	#
	# On some platforms there are 2 headers packages and both must be
	# installed. Here's an example on AWS:
	#  - linux-headers-5.3.0-1030-aws_5.3.0-1030.32~18.04.1_amd64.deb
	#  - linux-aws-5.3-headers-5.3.0-1030_5.3.0-1030.32~18.04.1_all.deb
	#
	local pkg
	for pkg in "${_RET_LIST[@]}"; do
		logmust install_pkgs "$DEPDIR/$pkg/"*-headers-*.deb
	done

	#
	# Verify that headers are installed for all kernel versions
	# stored in KERNEL_VERSIONS
	#
	local kernel
	for kernel in $KERNEL_VERSIONS; do
		logmust dpkg-query -l "linux-headers-$kernel" >/dev/null
	done
}

#
# Install kernel dbgsym packages for all target kernels.
# The kernel packages are fetched from S3.
#
function install_kernel_dbgsyms() {
	logmust determine_target_kernels
	check_env KERNEL_VERSIONS DEPDIR

	logmust list_linux_kernel_packages
	# Note: linux packages returned in _RET_LIST

	local pkg
	for pkg in "${_RET_LIST[@]}"; do
		logmust install_pkgs "$DEPDIR/$pkg/"linux-image-*dbgsym*.ddeb
	done

	#
	# Verify that headers are installed for all kernel versions
	# stored in KERNEL_VERSIONS
	#
	local kernel
	for kernel in $KERNEL_VERSIONS; do
		logmust dpkg-query -l "linux-image-$kernel-dbgsym*" >/dev/null
	done
}

function install_kernel_headers_and_dbgsyms() {
	logmust install_kernel_headers
	logmust install_kernel_dbgsyms

	#
	# Additionally, we add these symlinks so that kernel module builds will
	# be able to generate BTF information, as they look for the "vmlinux" file
	# in the kernel header directory.
	#
	local kernel
	for kernel in $KERNEL_VERSIONS; do
		logmust sudo ln -sf "/usr/lib/debug/boot/vmlinux-$kernel" "/usr/src/linux-headers-$kernel/vmlinux"
	done
}

function delphix_revision() {
	#
	# We use "delphix" in the default revision to make it easy to find all
	# packages built by delphix installed on an appliance. This will be used
	# along with DEFAULT_REVISION to get the full revision.
	#
	# We choose a timestamp as the second part since we want each package
	# built to have a unique value for its full version, as new packages
	# with the same full version as already installed ones would be skipped
	# during an upgrade.
	#
	# Note that having revision numbers increasing monotonically is a
	# requirement during regular upgrades. This is not a hard requirement for
	# Delphix Appliance upgrades, however we prefer keeping things in-line
	# with the established conventions.
	#
	echo "delphix.$(date '+%Y.%m.%d.%H.%M')"
}

function compare_versions() {
	dpkg --compare-versions "$@"
}

function determine_dependencies_base_url() {
	[[ -n "$DEPENDENCIES_BASE_URL" ]] && return

	check_env DEFAULT_GIT_BRANCH

	#
	# The location of package artifacts depends on whether a release tag
	# or a development branch is being built.
	#
	if is_release_branch; then
		local version="${DEFAULT_GIT_BRANCH#release/}"
		local url="s3://release-de-images/internal-artifacts/$version"
		local suv
		suv="$(aws s3 ls "$url/" | awk '{print $2}' | tr -d '/' | sort -V | tail -n 1)"
		[[ -n "$suv" ]] || die "No artifacts found at $url"
		DEPENDENCIES_BASE_URL="$url/$suv/input-artifacts/combined-packages/packages"
	else
		DEPENDENCIES_BASE_URL="s3://snapshot-de-images/builds/$JENKINS_OPS_DIR/linux-pkg/$DEFAULT_GIT_BRANCH/build-package"
	fi

	#
	# Make sure the dependencies directory actually exists
	#
	logmust aws s3 ls "$DEPENDENCIES_BASE_URL"

	export DEPENDENCIES_BASE_URL
}

function get_package_dependency_s3_url() {
	local dep="$1"

	logmust determine_dependencies_base_url

	#
	# The base url can be either pointing to:
	# 1. Input artifacts from a previously built Delphix Appliance, or
	# 2. A build-package/<package> directory that stores many builds of the
	#    same package and a "latest" link that points to the latest build
	#    for that package.
	# In case 1 the artifacts are right there under the base url, whereas
	# in case 2 we first need to dereference the "latest" link.
	#
	# The user can provide their own DEPENDENCIES_BASE_URL based on a
	# previous Delphix Appliance build or have it determined automatically
	# by determine_dependencies_base_url() based on the product branch
	# being built.
	#
	if [[ "$DEPENDENCIES_BASE_URL" == */build-package ]]; then
		local s3url="$DEPENDENCIES_BASE_URL/$dep/post-push"
		(logmust aws s3 cp --only-show-errors "$s3url/latest" .) ||
			die "Artifacts for dependency '$dep' missing." \
				"Dependency must be built first."
		logmust cat latest
		local bucket="${DEPENDENCIES_BASE_URL#s3://}"
		bucket=${bucket%%/*}
		s3url="s3://$bucket/$(cat latest)"
		logmust rm latest
		_RET="$s3url"
	else
		_RET="$DEPENDENCIES_BASE_URL/$dep"
	fi
}

#
# Fetch artifacts from S3 for all packages listed in PACKAGE_DEPENDENCIES which
# is defined in the package's config.
#
function fetch_dependencies() {
	local source="$1"
	export DEPDIR="$WORKDIR/dependencies"
	logmust mkdir "$DEPDIR"
	logmust cd "$DEPDIR"

	if [[ -z "$PACKAGE_DEPENDENCIES" ]]; then
		echo "Package has no linux-pkg dependencies to fetch."
		return
	fi

	local dep s3urlvar s3url
	for dep in $PACKAGE_DEPENDENCIES; do
		echo "Fetching artifacts for dependency '$dep' ..."
		get_package_prefix "$dep"
		case "$source" in
		"local")
			logmust cp -r "$WORKDIR/../../$dep/tmp/artifacts/" "$dep/"
			;;
		"s3")
			s3urlvar="${_RET}_S3_URL"
			if [[ -n "${!s3urlvar}" ]]; then
				s3url="${!s3urlvar}"
				echo "S3 URL of package dependency '$dep' provided externally"
				echo "$s3urlvar=$s3url"
			else
				logmust get_package_dependency_s3_url "$dep"
				s3url="$_RET"
			fi
			[[ "$s3url" != */ ]] && s3url="$s3url/"
			logmust mkdir "$dep"
			logmust aws s3 ls "$s3url"
			logmust aws s3 cp --only-show-errors --recursive "$s3url" "$dep/"
			echo_bold "Fetched artifacts for '$dep' from $s3url"
			PACKAGE_DEPENDENCIES_METADATA="${PACKAGE_DEPENDENCIES_METADATA}$dep: $s3url\\n"
			;;
		*)
			die "invalid source parameter specified: '$source'"
			;;
		esac
	done
}

#
# Run git fetch with the passed arguments. Git url must be passed as first
# argument. If FETCH_GIT_TOKEN is set and this is a github repository
# then pass-in the token when fetching.
#
function git_fetch_helper() {
	local orig_url="$1"
	local git_url="$1"
	local label=''
	shift

	if [[ -n "$FETCH_GIT_TOKEN" ]] &&
		[[ "$git_url" == https://github.com/* ]]; then
		git_url="${git_url/https:\/\//https:\/\/${FETCH_GIT_TOKEN}@}"
		label='[token passed]'
	fi
	echo "Running: $label git fetch $orig_url $*"
	#
	# Don't die here; logmust callers still fail-hard via logmust's own
	# error handling, while non-logmust callers (e.g. fetch_repo_from_git's
	# upstream-branch bootstrap fallback) can react to the failure.
	#
	git fetch "$git_url" "$@"
}

#
# Run git fetch with the passed arguments. Git url must be passed as first
# argument. If FETCH_GIT_TOKEN is set and this is a github repository
# then pass-in the token when fetching.
#
function git_branch_exists() {
	local git_url="$1"
	local label=''
	shift

	if [[ -n "$FETCH_GIT_TOKEN" ]] &&
		[[ "$git_url" == https://github.com/* ]]; then
		git_url="${git_url/https:\/\//https:\/\/${FETCH_GIT_TOKEN}@}"
		label='[token passed]'
	fi
	git fetch "$git_url" "$@" --no-tags --depth=1 || return 1
}

#
# Fetch package repository into $WORKDIR/repo
#
function fetch_repo_from_git() {
	check_env PACKAGE_GIT_URL PACKAGE_GIT_BRANCH DEFAULT_GIT_BRANCH

	logmust mkdir "$WORKDIR/repo"
	logmust cd "$WORKDIR/repo"
	logmust git init

	#
	# If we are updating the package, we need to fetch both the
	# main branch and the upstream branch with their histories.
	# Otherwise just get the latest commit of the main branch.
	#
	if [[ "$DO_UPDATE_PACKAGE" == "true" ]]; then
		logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
			"+$PACKAGE_GIT_BRANCH:repo-HEAD"
		#
		# Try to fetch the upstream-tracking branch from the package's
		# own repository. If it doesn't exist yet -- first time the
		# package is being synced -- bootstrap upstream-HEAD directly
		# from the third-party upstream specified by UPSTREAM_GIT_URL /
		# UPSTREAM_GIT_BRANCH, and mark upstream-updated so
		# sync-with-upstream.sh pushes upstream-HEAD back to origin,
		# creating the upstreams/<branch> branch in the process.
		#
		if git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
			"+upstreams/$DEFAULT_GIT_BRANCH:upstream-HEAD"; then
			:
		elif [[ -n "$UPSTREAM_GIT_URL" && -n "$UPSTREAM_GIT_BRANCH" ]]; then
			echo "NOTE: upstreams/$DEFAULT_GIT_BRANCH does not exist" \
				"on $PACKAGE_GIT_URL; bootstrapping upstream-HEAD" \
				"from $UPSTREAM_GIT_URL ($UPSTREAM_GIT_BRANCH)."
			logmust git_fetch_helper "$UPSTREAM_GIT_URL" --no-tags \
				"+$UPSTREAM_GIT_BRANCH:upstream-HEAD"
			logmust touch "$WORKDIR/upstream-updated"
		else
			die "Failed to fetch upstreams/$DEFAULT_GIT_BRANCH from" \
				"$PACKAGE_GIT_URL and UPSTREAM_GIT_URL/" \
				"UPSTREAM_GIT_BRANCH are not set to bootstrap from."
		fi
		logmust git show-ref repo-HEAD
		logmust git show-ref upstream-HEAD
	else

		#
		# The "os-upgrade" branch is used for Ubuntu version upgrade projects. As such, the following
		# block of code will only be executed if the `PACKAGE_GIT_BRANCH` is set to "os-upgrade". Not
		# all product repositories are required to have an "os-upgrade" branch, because some packages
		# will successfully build on the new Ubuntu version as well as the old without any changes.
		# To allow engineering to create the "os-upgrade" branch only in repositories that require
		# changes for the new Ubuntu version, this block of code will check if the "os-upgrade" branch
		# exists in the repository and if it does not, it will build off of "develop" instead.
		#
		if [[ "$PACKAGE_GIT_BRANCH" == "os-upgrade" ]]; then
			git_branch_exists "$PACKAGE_GIT_URL" "$PACKAGE_GIT_BRANCH"
			branch_exists=$?
			if [[ $branch_exists -ne 0 ]]; then
				echo "NOTE: The branch 'os-upgrade' does not exist for this repo. Using 'develop' instead."
				PACKAGE_GIT_BRANCH="develop"
			fi
		fi

		logmust git_fetch_helper "$PACKAGE_GIT_URL" --no-tags \
			"+$PACKAGE_GIT_BRANCH:repo-HEAD" --depth=1
		logmust git show-ref repo-HEAD
	fi

	logmust git checkout repo-HEAD
}

function generate_commit_message_from_dsc() {
	local dsc
	shopt -s failglob
	dsc=$(echo "$WORKDIR/source/$UPSTREAM_SOURCE_PACKAGE"*.dsc)
	shopt -u failglob

	rm -f "$WORKDIR/commit-message"
	grep -E '^Version:' "$dsc" >"$WORKDIR/commit-message"
	echo "" >>"$WORKDIR/commit-message"
	cat "$dsc" >>"$WORKDIR/commit-message"
}

#
# Fetches branch upstreams/<branch> from our default repository into local
# upstream-HEAD branch, then attempts to update it with changes from
# the source package UPSTREAM_SOURCE_PACKAGE, fetched from apt.
#
function update_upstream_from_source_package() {
	check_env PACKAGE_GIT_BRANCH UPSTREAM_SOURCE_PACKAGE

	#
	# Fetch the source package into source/
	#
	logmust mkdir "$WORKDIR/source"
	logmust cd "$WORKDIR/source"
	logmust apt-get source "$UPSTREAM_SOURCE_PACKAGE"

	#
	# Checkout the upstream branch from our repository, and delete all
	# files.
	#
	logmust cd "$WORKDIR/repo"
	logmust git checkout -q upstream-HEAD
	logmust git rm -qrf .
	logmust git clean -qfxd

	#
	# Deploy the files from the source package on top of our repo.
	#
	logmust cd "$WORKDIR"
	shopt -s dotglob failglob
	logmust mv source/"$UPSTREAM_SOURCE_PACKAGE"*/* repo/
	shopt -u dotglob failglob

	#
	# Check if there are any changes. If so then commit them, and put the
	# source package description as the commit message.
	#
	logmust cd "$WORKDIR/repo"
	logmust git add -f .
	if git diff --cached --quiet; then
		echo "NOTE: upstream for $PACKAGE is already up-to-date."
	else
		logmust generate_commit_message_from_dsc
		logmust git commit -F "$WORKDIR/commit-message"
		logmust git show-ref upstream-HEAD

		logmust touch "$WORKDIR/upstream-updated"
	fi

	logmust cd "$WORKDIR"
}

#
# Fetches branch upstreams/<branch> from our default repository into local
# upstream-HEAD branch, then attempts to update it with changes from
# the remote repository specified by UPSTREAM_GIT_URL and UPSTREAM_GIT_BRANCH.
#
function update_upstream_from_git() {
	check_env UPSTREAM_GIT_URL UPSTREAM_GIT_BRANCH
	logmust cd "$WORKDIR/repo"

	#
	# checkout our local branch that tracks upstream.
	#
	logmust git checkout -q upstream-HEAD

	#
	# Fetch updates from third-party upstream repository.
	#
	logmust git_fetch_helper "$UPSTREAM_GIT_URL" "$UPSTREAM_GIT_BRANCH"

	#
	# Compare third-party upstream repository to our local snapshot of the
	# upstream repository.
	#
	if git diff --quiet FETCH_HEAD upstream-HEAD; then
		echo "NOTE: upstream for $PACKAGE is already up-to-date."
	else
		#
		# Note we do --ff-only here which will fail if upstream has
		# been rebased. We always want this behaviour as a rebase
		# is not something that maintainers usually do and if they do
		# then we definitely want to be notified.
		#
		logmust git merge --no-edit --ff-only --no-stat FETCH_HEAD
		logmust git show-ref upstream-HEAD

		logmust touch "$WORKDIR/upstream-updated"
	fi

	logmust cd "$WORKDIR"
}

#
# Returns true if upstreams/<branch> needs to be merged into <branch> for the
# active package, where <branch> is the branch being updated, i.e.
# DEFAULT_GIT_BRANCH.
#
function is_merge_needed() {
	local repo_ref="refs/heads/repo-HEAD"
	local upstream_ref="refs/heads/upstream-HEAD"

	logmust pushd "$WORKDIR/repo"
	check_git_ref "$upstream_ref" "$repo_ref"

	if git merge-base --is-ancestor "$upstream_ref" "$repo_ref"; then
		echo "Upstream is already merged into repo-HEAD"
		_RET=false
	else
		_RET=true
	fi
	logmust popd
}

#
# Default function for merging upstreams/<branch> into <branch>, where <branch>
# is the branch being updated, i.e. DEFAULT_GIT_BRANCH. Note that this function
# does not actually look at the upstream repository itself, but relies on
# local branches repo-HEAD and upstream-HEAD to be present.
#
# If merge was needed, file $WORKDIR/repo-updated is created and previous tip
# of <branch> is saved in repo-HEAD-saved. The repo-updated file lets the
# caller (typically Jenkins) know if a merge was necessary. The repo-HEAD-saved
# ref should be compared to the remote branch when it is time to push the
# merge; if they differ it means that the remote branch was modified and
# so the merge should be aborted -- this can happen if a PR was merged by a
# developer while auto-update was running.
#
function merge_with_upstream_default() {
	local repo_ref="refs/heads/repo-HEAD"
	local upstream_ref="refs/heads/upstream-HEAD"

	logmust cd "$WORKDIR/repo"
	check_git_ref "$upstream_ref" "$repo_ref"

	logmust git checkout -q repo-HEAD

	if git merge-base --is-ancestor "$upstream_ref" HEAD; then
		echo "NOTE: $PACKAGE is already up-to-date with upstream."
		return 0
	fi

	#
	# Do a backup of the repo-HEAD branch so that it can be compared to the
	# remote when time comes to do a push.
	#
	logmust git branch repo-HEAD-saved

	logmust git merge --no-edit --no-stat "$upstream_ref"
	logmust git show-ref repo-HEAD
	logmust touch "$WORKDIR/repo-updated"
}

#
# Check if git credentials are set for pushing update. If running in
# interactive mode, it will prompt the user for credentials if they are not
# provided in env.
#
function check_git_credentials_set() {
	if [[ -z "$PUSH_GIT_TOKEN" ]] && [[ -z "$PUSH_GIT_USER" || -z "$PUSH_GIT_PASSWORD" ]]; then
		if [[ -t 1 ]]; then
			if [[ "$DRYRUN" == "false" ]]; then
				echo_bold "WARNING: this is NOT a dry-run, you are pushing to" \
					"a production branch"
			fi
			echo "Please enter git token to push to remote ($DEFAULT_PACKAGE_GIT_URL)."
			read -r -s -p "Token: " PUSH_GIT_TOKEN
			export PUSH_GIT_TOKEN
		else
			die "PUSH_GIT_TOKEN or PUSH_GIT_USER & PUSH_GIT_PASSWORD must be set."
		fi
	fi
}

#
# Push a local ref to a remote ref of the default remote repository for the
# package.
#
function push_to_remote() {
	local local_ref="$1"
	local remote_ref="$2"
	local force="${3:-false}"

	local flags=""
	$force && flags="-f"

	logmust check_git_credentials_set

	check_env DEFAULT_PACKAGE_GIT_URL

	local git_url_with_creds
	local git_url_with_fake_creds

	if [[ -n "$PUSH_GIT_TOKEN" ]]; then
		git_url_with_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_TOKEN}@}"
		git_url_with_fake_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_TOKEN:0:4}******@}"
	elif [[ -n "$PUSH_GIT_USER" && -n "$PUSH_GIT_PASSWORD" ]]; then
		git_url_with_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:${PUSH_GIT_PASSWORD}@}"
		git_url_with_fake_creds="${DEFAULT_PACKAGE_GIT_URL/https:\/\//https:\/\/${PUSH_GIT_USER}:******@}"
	else
		die "Either PUSH_GIT_TOKEN or PUSH_GIT_USER & PUSH_GIT_PASSWORD must be provided"
	fi

	logmust cd "$WORKDIR/repo"
	check_git_ref "$local_ref"

	echo "RUNNING: git push $flags $git_url_with_fake_creds $local_ref:$remote_ref"
	git push $flags "$git_url_with_creds" "$local_ref:$remote_ref" ||
		die "push failed"
}

#
# Creates a new changelog entry for the package with the appropriate fields.
# If no changelog file exists, source package name can be passed in first arg.
#
function set_changelog() {
	check_env PACKAGE_REVISION
	local final_version

	#
	# If the name of the source package isn't passed in as a parameter,
	# then deduce it. If there's a debian/control file that specifies that
	# package name, then use it. Otherwise, default to the name of the
	# linux-pkg directory name. This can't always be the default because
	# for some packages, those are different.  For example, the
	# challenge-response linux-pkg directory generates the
	# pam-challenge-response debian package.
	#
	if [[ -n $1 ]]; then
		src_package=$1
	elif [[ -f debian/control ]]; then
		src_package=$(awk '/^Source:/ { print $2 }' debian/control)
	fi
	src_package=${src_package:-$PACKAGE}

	#
	# If PACKAGE_VERSION hasn't been set already, then retrieve it from
	# The changelog file. If the changelog file doesn't exist, which
	# could be the case for in-house packages where we do not care about
	# the version, then default to "1.0.0".
	#
	if [[ -z "$PACKAGE_VERSION" ]]; then
		if [[ -f debian/changelog ]]; then
			PACKAGE_VERSION="$(logmust dpkg-parsechangelog -S Version)"
		else
			PACKAGE_VERSION=1.0.0
		fi
	fi

	logmust export DEBEMAIL="Delphix Engineering <eng@delphix.com>"
	if [[ "${PACKAGE_VERSION}" =~ '-' ]]; then
		final_version="${PACKAGE_VERSION}+${PACKAGE_REVISION}"
	else
		final_version="${PACKAGE_VERSION}-1${PACKAGE_REVISION}"
	fi
	if [[ -f debian/changelog ]]; then
		# update existing changelog
		# We use a + or - to easily visualize the separation
		# of the delphix revision. If there is no debian revision
		# then we add a debian revision using - as the separator,
		# If there is already a debian revision, we use + as the
		# separator and update the revision.

		logmust dch -b -v "$final_version" \
			"Automatically generated changelog entry."
	else
		# create new changelog
		logmust dch --create --package "$src_package" \
			-v "$final_version" \
			"Automatically generated changelog entry."
	fi
}

#
# Default dpkg_buildpackage function for building packages. Before running the
# build, it updates the version of the package in the changelog.
#
function dpkg_buildpackage_default() {
	logmust cd "$WORKDIR/repo"
	logmust set_changelog
	logmust dpkg-buildpackage -b -us -uc
	logmust cd "$WORKDIR/"
	logmust mv ./*deb artifacts/
}

#
# Returns the default (usually latest) kernel version for a given platform.
# Result is placed into _RET.
#
function get_kernel_version_for_platform_from_apt() {
	local platform="$1"
	local package

	#
	# For each supported platform, Ubuntu provides a 'linux-image-PLATFORM'
	# meta-package. This meta-package has a dependency on the default linux
	# image for that particular platform. For instance, Ubuntu has a
	# meta-package for AWS called 'linux-image-aws', which depends on
	# package 'linux-image-4.15.0-1027-aws'. The latter is the linux image
	# for kernel version '4.15.0-1027-aws'. We use this dependency to figure
	# out the default kernel version for a given platform.
	#
	# The "generic" platform is a special case, since we want to use the
	# hwe kernel image instead of the regular generic image.
	#
	# Note that while the default kernel is usually also the latest
	# available, it is not always the case.
	#
	if [[ "$platform" == generic ]]; then
		package="linux-image-${platform}-hwe-24.04"
	else
		package="linux-image-${platform}"
	fi

	if [[ "$(apt-cache show --no-all-versions "$package" \
		2>/dev/null | grep Depends)" =~ linux-image-([^,]*-${platform}) ]]; then
		_RET=${BASH_REMATCH[1]}
		return 0
	else
		die "failed to determine default kernel version for platform" \
			"'${platform}'"
	fi
}

#
# Given a kernel version, fetch all necessary linux kernel packages
# into WORKDIR/artifacts. Also store kernel version into KERNEL_VERSION.
#
function fetch_kernel_from_apt_for_version() {
	local kernel_version="$1"

	logmust cd "$WORKDIR/artifacts"
	logmust apt-get download \
		"linux-image-${kernel_version}" \
		"linux-image-${kernel_version}-dbgsym" \
		"linux-modules-${kernel_version}" \
		"linux-headers-${kernel_version}" \
		"linux-tools-${kernel_version}"

	#
	# For the azure kernel, we also want to get the linux-cloud-tools
	# package. Not that we cannot do this indiscriminately since some
	# kernel flavors do not come with a linux-cloud-tools package.
	#
	if [[ "$kernel_version" == *azure ]]; then
		logmust apt-get download "linux-cloud-tools-${kernel_version}"
	fi

	#
	# Fetch direct dependencies of the downloaded debs. Some of those
	# dependencies have a slightly different naming scheme than the other
	# kernel packages.
	#
	local deb dep deps
	for deb in *deb; do
		deps=$(dpkg-deb -f "$deb" Depends | tr -d ' ' | tr ',' ' ') ||
			die "failed to get dependencies for $deb"
		for dep in $deps; do
			case "$dep" in
			*-headers-* | *-tools-* | *-dbgsym)
				logmust apt-get download "$dep"
				;;
			esac
		done
	done

	echo "$kernel_version" >KERNEL_VERSION
}

#
# Find latest linux kernel available in apt for the given platform, and
# download all the necessary linux-kernel packages.
#
function fetch_kernel_from_apt_for_platform() {
	local platform="$1"

	local kernel_version
	logmust get_kernel_version_for_platform_from_apt "$platform"
	kernel_version="$_RET"

	logmust fetch_kernel_from_apt_for_version "$kernel_version"
}

#
# Fetch linux kernel packages from apt for the given kernel version. Also
# fetch the pre-built linux-modules package from artifactory. The pre-built
# package should have the same name as the one downloaded from apt but
# a higher revision number so that it will be picked over the default one
# downloaded from apt during the build of the appliance.
#
function fetch_kernel_from_artifactory() {
	local kernel_version="$1"
	local artifactory_deb="$2"

	logmust fetch_kernel_from_apt_for_version "$kernel_version"

	local url="http://artifactory.delphix.com/artifactory"
	url="$url/linux-pkg/linux-prebuilt/${artifactory_deb}"

	logmust cd "$WORKDIR/artifacts"
	fetch_file_from_artifactory "$url"
}

function fetch_file_from_artifactory() {
	local url="$1"
	local sha256_expected="$2"
	local file sha256_actual

	[[ -n "$url" ]] || die "url argument is missing."
	file="$(basename "$url")"

	logmust wget -nv "$url" -O "$file"

	if [[ -n "$sha256_expected" ]]; then
		sha256_actual="$(sha256sum "$file" | awk '{print $1}')"
		if [[ "$sha256_expected" != "$sha256_actual" ]]; then
			die "SHA256 mismatch. Expected: $sha256_expected," \
				"Actual: $sha256_actual"
		fi
	fi
}

#
# Determine which kernel versions to build modules for and store
# the value into KERNEL_VERSIONS, unless it is already set.
#
# We determine the target kernel versions based on the kernel package
# dependencies fetched through fetch_dependencies().
#
function determine_target_kernels() {
	if [[ -n "$KERNEL_VERSIONS" ]]; then
		echo_bold "Kernel versions to use to build modules:"
		echo_bold "  $KERNEL_VERSIONS"
		return 0
	fi

	[[ -n "$DEPDIR" ]] || die "determine_target_kernels() can only be" \
		"called after fetch_dependencies() stage has run."

	logmust list_linux_kernel_packages
	# note: list of kernel packages returned in _RET_LIST

	local pkg kernel
	for pkg in "${_RET_LIST[@]}"; do
		logmust test -d "$DEPDIR/$pkg"
		#
		# When Linux kernel packages are built, they must store the
		# kernel version into a file named 'KERNEL_VERSION'.
		#
		(logmust test -f "$DEPDIR/$pkg/KERNEL_VERSION") ||
			die "KERNEL_VERSION file missing from dependency '$pkg'"
		kernel="$(cat "$DEPDIR/$pkg/KERNEL_VERSION")"
		[[ -n "$kernel" ]] || die "invalid value '$kernel'" \
			"in $DEPDIR/$pkg/KERNEL_VERSION"
		KERNEL_VERSIONS="$KERNEL_VERSIONS $kernel"
		KERNEL_VERSIONS_METADATA="${KERNEL_VERSIONS_METADATA}${pkg}: ${kernel}\\n"
	done

	echo_bold "Kernel versions to use to build modules:"
	echo_bold "  $KERNEL_VERSIONS"
}

#
# Store git-related build info for the package after the build is done.
# Note that some of this metadata is used by the Jenkins build so be careful
# when modifying it.
#
function store_git_info() {
	logmust pushd "$WORKDIR/repo"
	local git_hash
	git_hash="$(git rev-parse HEAD)" || die "Failed retrieving git hash"
	echo "$git_hash" >"$WORKDIR/artifacts/GIT_HASH"

	cat <<-EOF >>"$WORKDIR/artifacts/BUILD_INFO"
		Git hash: $git_hash
		Git repo: $PACKAGE_GIT_URL
		Git branch: $PACKAGE_GIT_BRANCH
	EOF
	logmust popd
}

#
# Store build info metadata for the package after the build is done.
# Note that some of this metadata is used by the Jenkins build so be careful
# when modifying it.
#
function store_build_info() {
	if [[ -d "$WORKDIR/repo/.git" ]]; then
		logmust store_git_info
	fi

	if [[ -n "$KERNEL_VERSIONS_METADATA" ]]; then
		echo -ne "$KERNEL_VERSIONS_METADATA" >"$WORKDIR/artifacts/KERNEL_VERSIONS" ||
			die 'Failed to store kernel versions metadata'
	fi

	if [[ -n "$PACKAGE_DEPENDENCIES_METADATA" ]]; then
		echo -ne "$PACKAGE_DEPENDENCIES_METADATA" >"$WORKDIR/artifacts/PACKAGE_DEPENDENCIES" ||
			die 'Failed to store package dependencies metadata'
	fi

	if [[ -f "$TOP/PACKAGE_MIRROR_URL_MAIN" ]]; then
		logmust cp "$TOP/PACKAGE_MIRROR_URL_MAIN" "$WORKDIR/artifacts/"
	fi

	if [[ -f "$TOP/PACKAGE_MIRROR_URL_SECONDARY" ]]; then
		logmust cp "$TOP/PACKAGE_MIRROR_URL_SECONDARY" "$WORKDIR/artifacts/"
	fi
}

#
# Generate a CycloneDX SBOM for each of this package's built
# .deb(s) by running Syft against it. Only packages that bundle
# third-party composition (jars, npm, wheels, Rust crates, ...) opt in
# via SBOM_DEEP_SCAN="true" in their config.sh -- everything else is
# left as a flat pkg:deb component by appliance-build's base chroot
# scan, so a deb scan here would add nothing. Each <deb-filename>.cdx.json
# is dropped in $WORKDIR/artifacts/ alongside the .deb it describes --
# a strict 1:1 mapping, no merging across a package's .deb(s) -- where
# it's picked up by the same S3 sync as every other build artifact, no
# separate upload path needed.
#
#
# Rewrite a Syft-generated CycloneDX document in place, dropping what should
# not be published and collapsing Syft's per-location duplicates. See
# resources/sanitize-sbom.jq for what is removed and why.
#
# Skipped entirely when CYCLONEDX_FILTERING is "false", which leaves the raw
# Syft output -- including syft:metadata:virtualPath, the only field that
# records where inside the package a component was found. That is of no use to
# a customer but is how we work out where an unexpected component came from,
# so it stays available on demand.
#
function sanitize_sbom() {
	local sbom_file="$1"
	local tmp

	check_env TOP
	[[ -f "$sbom_file" ]] || die "sanitize_sbom: '$sbom_file' does not exist."

	tmp="$(logmust mktemp)"

	#
	# jq cannot edit a file in place, so write to a temporary file and
	# move it over only once jq has succeeded. Redirecting straight back
	# into the input would truncate it before jq had read it.
	#
	logmust jq -f "$TOP/resources/sanitize-sbom.jq" "$sbom_file" >"$tmp"
	logmust mv "$tmp" "$sbom_file"
}

function generate_sbom() {
	if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then
		return 0
	fi

	local debs=("$WORKDIR/artifacts/"*.deb)
	if [[ ! -e "${debs[0]}" ]]; then
		die "SBOM_DEEP_SCAN is set but no .deb was found in" \
			"'$WORKDIR/artifacts'"
	fi

	#
	# syft/cyclonedx-cli are part of the generic build tooling installed
	# by setup.sh (install_sbom_tools()) before any package is built,
	# rather than something each SBOM_DEEP_SCAN package declares for
	# itself -- this is a default hook, so what it needs is its own
	# concern, not its callers'. That install is best-effort, so check
	# here rather than letting "command not found" surface from the
	# middle of a scan.
	#
	local tool
	for tool in syft cyclonedx-cli; do
		command -v "$tool" >/dev/null ||
			die "'$tool' is not installed, so no SBOM can be" \
				"generated for '$PACKAGE'. It is provisioned by" \
				"install_sbom_tools() in setup.sh; check that" \
				"run's output for why it was skipped."
	done

	#
	# jq comes from setup.sh's baseline package install rather than
	# install_sbom_tools(), but sanitize_sbom() is useless without it, so
	# fail here rather than part-way through a package's .debs.
	#
	if [[ "$CYCLONEDX_FILTERING" != "false" ]]; then
		command -v jq >/dev/null ||
			die "'jq' is not installed, so the SBOM for '$PACKAGE'" \
				"cannot be sanitized. Either install it (setup.sh" \
				"does so by default) or set CYCLONEDX_FILTERING" \
				"to 'false' to publish the raw Syft output."
	fi

	#
	# One SBOM file per .deb, not per package: a package that emits more
	# than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux,
	# etc.) gets one <deb-filename>.deb.cdx.json per .deb, each a
	# standalone document scoped to that .deb alone. No merging across
	# .debs -- keeps a strict 1:1 mapping between a .deb and its BOM,
	# with the .deb's own filename as the common prefix.
	#
	local deb
	for deb in "${debs[@]}"; do
		local sbom_file deb_version extract_dir
		sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json"
		#
		# Read the version back out of the .deb itself, rather than
		# relying on $PACKAGE_VERSION: by this point in the build,
		# $PACKAGE_VERSION may no longer hold the final,
		# revision-suffixed version set_changelog() wrote into the
		# package (e.g. it's empty for packages that don't set it
		# explicitly themselves, unlike syft/cyclonedx-cli's own
		# config.sh). dpkg-deb reads the actual, authoritative
		# version of the artifact being scanned.
		#
		deb_version="$(dpkg-deb -f "$deb" Version)"

		#
		# Scan the .deb's extracted payload rather than the .deb file.
		# "syft scan <file>.deb" only identifies the archive: its
		# deb-archive-cataloger reads the control metadata and emits a
		# single pkg:deb component, never descending into data.tar.*,
		# so none of the bundled jars/wheels/modules this SBOM
		# exists to capture are found. That produced valid but empty
		# documents -- one component for a 1.2GB application -- which
		# is no more than appliance-build's image-level dpkg scan
		# already gives for free. Extracting first is what the design
		# spec prescribed for exactly this case.
		#
		extract_dir="$(logmust mktemp -d)"
		logmust dpkg-deb -x "$deb" "$extract_dir"

		#
		# Full catalogers here, deliberately unlike appliance-build's
		# 95-generate-sbom.binary hook, which restricts to dpkg. That
		# restriction exists because a jar sitting somewhere on a whole
		# rootfs cannot be attributed to the package that placed it;
		# within a single package's own extracted payload everything
		# found belongs to that package by construction, which is the
		# entire point of scanning here rather than at image level.
		#
		# SYFT_FILE_METADATA_SELECTION=none suppresses Syft's default
		# per-file "file" component (with SHA-1/SHA-256 hashes and the
		# scanned path baked in) -- with a whole extracted payload to
		# walk that would otherwise emit an entry per file.
		#
		SYFT_FILE_METADATA_SELECTION=none logmust syft scan "dir:$extract_dir" \
			--source-name "$PACKAGE" \
			--source-version "$deb_version" \
			-o "cyclonedx-json@1.6=$sbom_file"

		logmust rm -rf "$extract_dir"

		#
		# Sanitize before validating, so that what is validated is
		# exactly what is published.
		#
		# Only the exact string "false" disables this. Anything else,
		# including unset, sanitizes: a typo should leave us with a
		# publishable document rather than silently shipping one that
		# records our internal paths.
		#
		if [[ "$CYCLONEDX_FILTERING" != "false" ]]; then
			logmust sanitize_sbom "$sbom_file"
		else
			echo_bold "CYCLONEDX_FILTERING is 'false': publishing" \
				"the raw Syft output for $(basename "$deb")."
		fi

		logmust cyclonedx-cli validate \
			--input-file "$sbom_file" \
			--input-format json \
			--input-version v1_6 \
			--fail-on-errors
	done
}

function set_secret_build_args() {
	_SECRET_BUILD_ARGS=()

	if [[ -n "${SECRET_DB_USE_JUMPBOX}" ]]; then
		_SECRET_BUILD_ARGS+=("-DSECRET_DB_USE_JUMPBOX=$SECRET_DB_USE_JUMPBOX")
	fi

	if [[ -n "${SECRET_DB_JUMP_BOX_HOST}" ]]; then
		_SECRET_BUILD_ARGS+=("-DSECRET_DB_JUMP_BOX_HOST=$SECRET_DB_JUMP_BOX_HOST")
	fi

	if [[ -n "${SECRET_DB_JUMP_BOX_USER}" ]]; then
		_SECRET_BUILD_ARGS+=("-DSECRET_DB_JUMP_BOX_USER=$SECRET_DB_JUMP_BOX_USER")
	fi

	if [[ -n "${SECRET_DB_JUMP_BOX_PRIVATE_KEY}" ]]; then
		if [[ ! -f "$SECRET_DB_JUMP_BOX_PRIVATE_KEY" ]]; then
			die "Jumpbox private key not found."
		fi
		_SECRET_BUILD_ARGS+=("-DSECRET_DB_JUMP_BOX_PRIVATE_KEY=$SECRET_DB_JUMP_BOX_PRIVATE_KEY")
	fi

	if [[ -n "${SECRET_DB_AWS_ENDPOINT}" ]]; then
		_SECRET_BUILD_ARGS+=("-DSECRET_DB_AWS_ENDPOINT=$SECRET_DB_AWS_ENDPOINT")
	fi

	if [[ -n "${SECRET_DB_AWS_PROFILE}" ]]; then
		_SECRET_BUILD_ARGS+=("-DSECRET_DB_AWS_PROFILE=$SECRET_DB_AWS_PROFILE")
	fi

	if [[ -n "${SECRET_DB_AWS_REGION}" ]]; then
		_SECRET_BUILD_ARGS+=("-DSECRET_DB_AWS_REGION=$SECRET_DB_AWS_REGION")
	fi
}

#
# Secure boot variables and functions
#
# S3 bucket containing keys and certs
# ./db subdirectory contains the db key and various certs:
#   .der is for signing modules like ZFS and connstat
#   .crt is for signing vmlinuz
#   signing_key.pem is the format expected by kernel build for signing its modules
#
# ./pub contains the auth files, secure boot enrollment certs.
#
S3_KEYS_URL="s3://secure-boot-keys-prod/release"
#
# The kernel build expects the signing_key.pem in this directory, i.e.
# CONFIG_MODULE_SIG_KEY is set to /var/tmp/sbkeys/signing_key.pem in
# resources/delphix_kernel_annotations
#
SB_KEYS_DIR="/var/tmp/sbkeys"
SBSIGN_KEY="$SB_KEYS_DIR/db.key"
SBSIGN_DER="$SB_KEYS_DIR/db.der"

function download_keys() {
	logmust mkdir -p $SB_KEYS_DIR
	logmust aws s3 cp --recursive "$S3_KEYS_URL/db/" $SB_KEYS_DIR
}

function delete_keys() {
	logmust rm -r $SB_KEYS_DIR
}

# Update DEBIAN/md5sum for package directory after
# some files were updated, i.e. secure-boot signed.
#
function update_md5sums() {
	pkg_dir=$1
	echo_bold "Updating md5sums for $pkg_dir"

	(
		cd "$pkg_dir" || exit
		: >DEBIAN/md5sums
		# print paths relative to root of package
		while IFS= read -r -d '' f; do
			rel="${f#./}"
			md5sum "$rel" >>DEBIAN/md5sums
		done < <(find . -type f ! -path './DEBIAN/*' ! -path './etc/depmod*' -print0)
	)
}

function repack_deb() {
	deb_name=$1
	deb_dir=$2
	temp_deb=$(mktemp /tmp/deb.XXXXXX)

	logmust fakeroot dpkg-deb -b "$deb_dir" "$temp_deb"
	logmust mv "$temp_deb" "$deb_name"
}

#
# Sign .ko files in the module list
#
function sign_modules() {
	deb_pkgs="$1"
	echo_bold "Signing $deb_pkgs"
	download_keys

	while IFS= read -r pkg; do
		echo_bold "Processing $pkg"
		temp_dir=$(mktemp -d -p "/var/tmp/")
		logmust fakeroot dpkg-deb -R "$pkg" "$temp_dir"

		#
		# Find and sign all .ko files in package. The find output is fed
		# in by process substitution rather than a pipe so that the loop
		# body runs in this shell: in a pipeline it would run in a
		# subshell, where logmust's die only exits the subshell and the
		# package would go on to be repacked with unsigned modules.
		#
		while IFS= read -r -d '' kernel_mod; do
			logmust kmodsign sha256 "$SBSIGN_KEY" "$SBSIGN_DER" "$kernel_mod" "$kernel_mod.signed"
			logmust mv "$kernel_mod.signed" "$kernel_mod"
			logmust modinfo -F signer "$kernel_mod"
		done < <(find "$temp_dir" -type f -name "*.ko" -print0)

		# Repack the .deb"
		update_md5sums "$temp_dir"
		repack_deb "$pkg" "$temp_dir"
	done <<<"$deb_pkgs"
	delete_keys
}