From 2ee9455879f80aa0e6720f39d4fc73dcf2bf381f Mon Sep 17 00:00:00 2001 From: Dave Page Date: Tue, 22 Sep 2026 11:23:51 +0100 Subject: [PATCH 1/2] Harden load-docs.sh against a hostile docs tarball The docs tarball is assembled by a CI runner and written into a tree that the upload key can reach, and the script unpacked it straight over /var/www/pgaweb/static/docs with an unquoted path and no checks at all, so the archive rather than the wrapper decided where those files landed. That is where the publishing wrapper's containment ended. The version argument is now checked for shape before it is interpolated into either the tarball path or the directory name, every variable is quoted, and the archive is vetted before anything is unpacked: every member must be a plain file or a directory, which rejects symlinks and hard links that would redirect a later member out of the tree, and every member must sit under the pgadmin4--docs directory, which rejects absolute paths and '..' traversal. Unpacking then happens into a staging directory alongside the live one, with --no-same-owner and --no-same-permissions, and the result is checked and moved into place only once it looks like the docs we asked for, so a bad tarball can no longer leave the site serving a half replaced tree. The comments explain what each check is for, since the next reader will otherwise have no way of knowing. Behaviour on a good tarball is unchanged apart from the loss of tar's verbose file listing, with the docloader call and the cache ban exactly as they were. --- tools/load-docs.sh | 110 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 5 deletions(-) diff --git a/tools/load-docs.sh b/tools/load-docs.sh index bc28157..492cd15 100755 --- a/tools/load-docs.sh +++ b/tools/load-docs.sh @@ -1,27 +1,127 @@ #!/bin/bash +set -e +set -o pipefail + +DOCS_ROOT=/var/www/pgaweb/static/docs +FTP_ROOT=/var/ftp/pgadmin4 + if [ $# -ne 1 ] then echo "usage: $0 " exit 1 fi +VERSION=$1 + +# The version is interpolated into the tarball path and into the name of the +# directory we unpack into, so its shape is checked before it is used for +# either, because a value carrying a slash, a '..' or a shell metacharacter +# would let whoever called us choose those paths rather than merely name a +# release. +if [[ ! ${VERSION} =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] +then + echo "Invalid version number (${VERSION}); expected something like 9.18 or 9.18.1." + exit 1 +fi + +TARBALL=${FTP_ROOT}/v${VERSION}/docs/pgadmin4-${VERSION}-docs.tar.gz +DOCS_DIR=pgadmin4-${VERSION}-docs + # Check the docs file exists -TARBALL=/var/ftp/pgadmin4/v${1}/docs/pgadmin4-${1}-docs.tar.gz -if [ ! -f ${TARBALL} ] +if [ ! -f "${TARBALL}" ] then echo "The source tarball (${TARBALL}) could not be found." exit 1 fi +# Everything below happens in a staging directory beside the live one, so that +# a tarball which turns out to be bad, or an unpacking that dies part way +# through, cannot leave the site serving a half replaced tree. It sits inside +# DOCS_ROOT so that moving the result into place is a rename rather than a copy +# of the best part of a gigabyte. +STAGING=$(mktemp -d "${DOCS_ROOT}/.load-docs.XXXXXX") +trap 'rm -rf "${STAGING}"' EXIT + +# The tarball is assembled by a CI runner and lands in a tree that an upload +# key can write to, so it is untrusted input, and an archive decides for itself +# where its contents are written. We therefore vet the member list before +# unpacking anything, rather than trusting whatever a given tar happens to do +# with a hostile archive, since that varies between implementations and +# versions and is not a control we own. The two listings also fail the script +# outright if the archive cannot be read from end to end. +tar -tzvf "${TARBALL}" > "${STAGING}/listing.verbose" +tar -tzf "${TARBALL}" > "${STAGING}/listing.names" + +# Every member must be a plain file or a directory, which rejects symlinks and +# hard links whose targets could redirect a later member out of the docs tree, +# along with devices and anything else that has no business in a docs tarball. +# A listing line we cannot make sense of fails this test as well, which is the +# right way round for it to fail. +if cut -c1 "${STAGING}/listing.verbose" | sort -u | grep -qv '^[-d]$' +then + echo "${TARBALL} holds members that are neither plain files nor directories." + exit 1 +fi + +# Every member must also sit under the one directory we expect, which rejects +# absolute paths and '..' traversal, and confirms the archive carries the +# single top level directory that the docloader below is pointed at. +while read -r MEMBER +do + case ${MEMBER} in + "${DOCS_DIR}"|"${DOCS_DIR}"/) + ;; + "${DOCS_DIR}"/*) + case ${MEMBER} in + */../*|*/..) + echo "${TARBALL} holds a member traversing out of ${DOCS_DIR} (${MEMBER})." + exit 1 + ;; + esac + ;; + *) + echo "${TARBALL} holds a member outside ${DOCS_DIR} (${MEMBER})." + exit 1 + ;; + esac +done < "${STAGING}/listing.names" + +rm -f "${STAGING}/listing.verbose" "${STAGING}/listing.names" + # Unpack the tarball -cd /var/www/pgaweb/static/docs -tar -zxvf ${TARBALL} +tar -xzf "${TARBALL}" -C "${STAGING}" --no-same-owner --no-same-permissions + +# Confirm we got out what we expected, rather than assuming that a tar which +# exited zero wrote the docs we are about to load and publish. The second check +# is belt and braces over the member vetting above, and catches a tar that +# creates something odd from an archive whose listing looked ordinary. +if [ ! -f "${STAGING}/${DOCS_DIR}/index.html" ] +then + echo "${TARBALL} did not unpack to ${DOCS_DIR} holding an index.html." + exit 1 +fi + +if [ -n "$(find "${STAGING}" ! -type d ! -type f -print -quit)" ] +then + echo "${TARBALL} unpacked something that is neither a plain file nor a directory." + exit 1 +fi + +# Swap the new docs into place, keeping the old copy until the new one has +# landed so that reloading a version we already hold does not leave a gap. +if [ -e "${DOCS_ROOT}/${DOCS_DIR}" ] +then + rm -rf "${DOCS_ROOT}/${DOCS_DIR}.replaced" + mv "${DOCS_ROOT}/${DOCS_DIR}" "${DOCS_ROOT}/${DOCS_DIR}.replaced" +fi +mv "${STAGING}/${DOCS_DIR}" "${DOCS_ROOT}/${DOCS_DIR}" +rm -rf "${DOCS_ROOT}/${DOCS_DIR}.replaced" # Load 'em cd /var/www/pgaweb source /usr/share/python3/pginfra-virtualenv-django52-py3/bin/activate -./manage.py docloader pgadmin4 ${1} pgadmin4-${1}-docs +./manage.py docloader pgadmin4 "${VERSION}" "${DOCS_DIR}" # Clear the cache sudo varnishadm "ban req.url ~ ." From 9a3856a425f80a08d2c764cdc49261976bedb612 Mon Sep 17 00:00:00 2001 From: Dave Page Date: Tue, 22 Sep 2026 11:23:51 +0100 Subject: [PATCH 2/2] Drop the tools that have moved to the pgadmin4 repository rebuild-apt-repo.sh and rebuild-yum-repo.sh are superseded by the parameterised versions at pkg/publish/ in pgadmin4, which are what the publishing wrapper actually runs; nothing here called them, and two copies of the repository signing logic is precisely the duplication the move to GitHub Actions set out to remove. sync-ftp-to-s3.py and purge_pgadmin_releases.sh operate on the download tree at /var/ftp/pgadmin4 and touch nothing in this Django application, so they belong beside the rest of the publishing machinery rather than here. The README described four scripts as being called by the Jenkins server, which is no longer true of any of them. The release time ones are now reached through the publishing wrapper from GitHub Actions, whilst update-docs.sh runs from cron on the web server and never had anything to do with Jenkins. The note about the AWS CLI goes with sync-ftp-to-s3.py, which was the only thing that used it. --- tools/README.md | 52 ++----- tools/purge_pgadmin_releases.sh | 266 -------------------------------- tools/rebuild-apt-repo.sh | 28 ---- tools/rebuild-yum-repo.sh | 18 --- tools/sync-ftp-to-s3.py | 156 ------------------- 5 files changed, 16 insertions(+), 504 deletions(-) delete mode 100755 tools/purge_pgadmin_releases.sh delete mode 100755 tools/rebuild-apt-repo.sh delete mode 100755 tools/rebuild-yum-repo.sh delete mode 100755 tools/sync-ftp-to-s3.py diff --git a/tools/README.md b/tools/README.md index 5ef9638..f4605e7 100644 --- a/tools/README.md +++ b/tools/README.md @@ -4,50 +4,30 @@ This directory contains various scripts used to manage www.pgadmin.org and pgAdm **Most are very system-specific; don't expect them to work for you!!** -**Note:** The AWS CLI is expected to be installed at /usr/bin/aws. +On the production web server, all scripts **must** be run as the *pgaupload* user. -Om the production web server, all scripts **must** be run as the *pgaupload* user. +## create_release.py -## create-release.py - -This script is called by the Jenkins server as part of the release process. It creates all the required objects in the -website database for a new release. +This script is called as part of the release process, through the `create-release` verb of the publishing wrapper +(`pkg/publish/pga-publish` in the pgadmin4 repo) which the GitHub Actions workflows reach over SSH. It creates all the +required objects in the website database for a new release. ## load-docs.sh -This script is called by the Jenkins server as part of the release process. It unpacks the docs for the version of -pgAdmin specified on the command line, and loads them into the website database. - -## purge-cache.sh - -This script is called by the Jenkins server as part of the release process. It purges the entire Varnish cache that -sits in front of Nginx. - -# rebuild-apt-repo.sh +This script is called as part of the release process, through the `load-docs` verb of the publishing wrapper. It +unpacks the docs for the version of pgAdmin specified on the command line, and loads them into the website database. -This script will rebuild the specified APT repo metadata, and re-sign everything. This is typically used if old -packages are manually removed. +Because the tarball it unpacks is built elsewhere and lands in a tree that an upload key can write to, the script +treats it as untrusted input and vets the member list before unpacking anything; the comments in the script explain +what each check is there to stop. -```bash -./rebuild-apt-repo.sh focal -``` - -# rebuild-yum-repo.sh - -This script will rebuild the specified YUM repo metadata, and re-sign everything. This is typically used if old -packages are manually removed. - -```bash -./rebuild-yum-repo.sh redhat rhel 8 -``` - -## sync-ftp-to-s3.py +## purge-cache.sh -This script is called by the Jenkins server as part of the release process. It will copy all the packages to the S3 -bucket that hosts https://pgadmin-archive.postgresql.org, and generate and sync HTML index pages for browsing. +This script is called as part of the release process, through the `purge-cache` verb of the publishing wrapper. It +purges the entire Varnish cache that sits in front of Nginx. ## update-docs.sh -This script is typically called from cron. It updates the git checkout of pgAdmin at -/var/www/pgaweb/static/docs/pgadmin4-dev, and then rebuilds the HTML docs which are subsequently loaded into the -website database as the 'dev' docs. +This script is run from cron on the web server, twice an hour, as the *pgaupload* user, and is not part of the release +process. It updates the git checkout of pgAdmin at /var/www/pgaweb/static/docs/pgadmin4-dev, and then rebuilds the HTML +docs which are subsequently loaded into the website database as the 'dev' docs. diff --git a/tools/purge_pgadmin_releases.sh b/tools/purge_pgadmin_releases.sh deleted file mode 100755 index 5605cc3..0000000 --- a/tools/purge_pgadmin_releases.sh +++ /dev/null @@ -1,266 +0,0 @@ -#!/bin/bash -# -# purge_pgadmin_releases.sh - Purge old pgAdmin releases -# -# Removes old pgAdmin version directories, APT packages, and YUM packages -# for a given major version, keeping only the specified version and later. -# -# Usage: purge_pgadmin_releases.sh [--live] -# -# Example: purge_pgadmin_releases.sh 9.11 -# This will purge versions 9.0 through 9.10 but NOT 8.x, 7.x, etc. -# -# By default runs in dry-run mode. Use --live to actually delete files. -# -# If an OS/distro repo directory only contains pgAdmin versions that would -# be purged, the most recent of those versions is kept so the repo is not -# left empty of that major version. - -set -euo pipefail - -BASE_DIR="/var/ftp/pgadmin4" -LIVE=0 - -usage() { - echo "Usage: $0 [--live] " - echo "" - echo " Purges all pgAdmin releases older than within the" - echo " same major version." - echo "" - echo " --live Actually delete files (default is dry-run)" - echo "" - echo " Example: $0 9.11" - echo " Purges 9.0 through 9.10, keeps 9.11 and later." - exit 1 -} - -# Parse arguments -while [[ $# -gt 0 ]]; do - case "$1" in - --live) - LIVE=1 - shift - ;; - -h|--help) - usage - ;; - -*) - echo "Error: Unknown option: $1" - usage - ;; - *) - VERSION="$1" - shift - ;; - esac -done - -if [[ -z "${VERSION:-}" ]]; then - echo "Error: Version argument is required." - usage -fi - -# Validate version format -if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+$'; then - echo "Error: Version must be in major.minor format (e.g. 9.11)" - exit 1 -fi - -TARGET_MAJOR="${VERSION%%.*}" -TARGET_MINOR="${VERSION#*.}" - -if [[ "$LIVE" -eq 1 ]]; then - echo "*** LIVE MODE - files will be deleted ***" -else - echo "*** DRY RUN MODE - no files will be deleted (use --live to delete) ***" -fi -echo "" -echo "Purging pgAdmin ${TARGET_MAJOR}.0 through ${TARGET_MAJOR}.$((TARGET_MINOR - 1))" -echo "" - -# Helper: execute or print an rm command -do_rm() { - if [[ "$LIVE" -eq 1 ]]; then - rm -rf "$@" - else - echo " Would remove: $*" - fi -} - -# Track totals -TOTAL_REMOVED=0 -TOTAL_KEPT=0 - -############################################################################### -# 1. Purge version directories (e.g. /var/ftp/pgadmin4/v9.0) -############################################################################### -echo "=== Version directories ===" -for minor in $(seq 0 $((TARGET_MINOR - 1))); do - VDIR="${BASE_DIR}/v${TARGET_MAJOR}.${minor}" - if [[ -d "$VDIR" ]]; then - do_rm "$VDIR" - TOTAL_REMOVED=$((TOTAL_REMOVED + 1)) - fi -done -echo "" - -############################################################################### -# 2. Purge APT packages -############################################################################### -echo "=== APT repositories ===" -for distro_dir in "${BASE_DIR}"/apt/*/; do - [[ -d "$distro_dir" ]] || continue - distro=$(basename "$distro_dir") - - # Find all binary directories under dists/pgadmin4/main/ - for bin_dir in "${distro_dir}"dists/pgadmin4/main/binary-*/; do - [[ -d "$bin_dir" ]] || continue - arch=$(basename "$bin_dir") - - # Collect pgadmin4 .deb files for the target major version in this dir - # File patterns: pgadmin4_{ver}_all.deb, pgadmin4-desktop_{ver}_amd64.deb, etc. - # Version in filename can be like "9.5" or "9.10-1.bookworm" - declare -A versions_in_dir=() - declare -A files_by_version=() - - while IFS= read -r -d '' deb_file; do - fname=$(basename "$deb_file") - # Extract version: pgadmin4[-something]_{VERSION}_{arch}.deb - # VERSION is like "9.5" or "9.10-1.bookworm" - pkg_ver=$(echo "$fname" | sed -E 's/^pgadmin4[^_]*_([0-9]+\.[0-9]+).*/\1/') - pkg_major="${pkg_ver%%.*}" - pkg_minor="${pkg_ver#*.}" - - if [[ "$pkg_major" == "$TARGET_MAJOR" ]]; then - versions_in_dir["$pkg_minor"]=1 - files_by_version["${pkg_minor}:${deb_file}"]="$deb_file" - fi - done < <(find "$bin_dir" -maxdepth 1 -name "pgadmin4*.deb" -print0 2>/dev/null) - - [[ ${#versions_in_dir[@]} -eq 0 ]] && unset versions_in_dir files_by_version && continue - - # Determine which minor versions to purge (< TARGET_MINOR) - purge_minors=() - keep_minors=() - for m in "${!versions_in_dir[@]}"; do - if [[ "$m" -lt "$TARGET_MINOR" ]]; then - purge_minors+=("$m") - else - keep_minors+=("$m") - fi - done - - # If ALL versions of this major would be purged, keep the highest - highest_purge="" - if [[ ${#keep_minors[@]} -eq 0 && ${#purge_minors[@]} -gt 0 ]]; then - for m in "${purge_minors[@]}"; do - if [[ -z "$highest_purge" || "$m" -gt "$highest_purge" ]]; then - highest_purge="$m" - fi - done - echo " [${distro}/${arch}] No versions >= ${TARGET_MAJOR}.${TARGET_MINOR} found; keeping ${TARGET_MAJOR}.${highest_purge}" - TOTAL_KEPT=$((TOTAL_KEPT + 1)) - fi - - # Remove files for purged versions - for key in "${!files_by_version[@]}"; do - m="${key%%:*}" - f="${key#*:}" - if [[ "$m" -lt "$TARGET_MINOR" && "$m" != "$highest_purge" ]]; then - do_rm "$f" - TOTAL_REMOVED=$((TOTAL_REMOVED + 1)) - fi - done - - unset versions_in_dir files_by_version - done -done -echo "" - -############################################################################### -# 3. Purge YUM/DNF packages -############################################################################### -echo "=== YUM repositories ===" -for vendor_dir in "${BASE_DIR}"/yum/*/; do - [[ -d "$vendor_dir" ]] || continue - vendor=$(basename "$vendor_dir") - - # Skip non-directory files (like the .noarch.rpm and README at the top level) - for os_dir in "${vendor_dir}"*/; do - [[ -d "$os_dir" ]] || continue - # Skip symlinks (e.g. fedora-41Client-x86_64 -> fedora-41-x86_64) - [[ -L "${os_dir%/}" ]] && continue - os=$(basename "$os_dir") - - # Skip repodata directories and non-repo items - [[ "$os" == "repodata" ]] && continue - - # Collect pgadmin4 RPMs for the target major version - declare -A yum_versions_in_dir=() - declare -A yum_files_by_version=() - - while IFS= read -r -d '' rpm_file; do - fname=$(basename "$rpm_file") - # Extract version: pgadmin4[-something]-{MAJOR}.{MINOR}-{release}.{dist}.{arch}.rpm - pkg_ver=$(echo "$fname" | sed -E 's/.*pgadmin4[^0-9]*([0-9]+\.[0-9]+)-.*/\1/') - pkg_major="${pkg_ver%%.*}" - pkg_minor="${pkg_ver#*.}" - - if [[ "$pkg_major" == "$TARGET_MAJOR" ]]; then - yum_versions_in_dir["$pkg_minor"]=1 - yum_files_by_version["${pkg_minor}:${rpm_file}"]="$rpm_file" - fi - done < <(find "$os_dir" -maxdepth 1 -name "pgadmin4*.rpm" -print0 2>/dev/null) - - [[ ${#yum_versions_in_dir[@]} -eq 0 ]] && unset yum_versions_in_dir yum_files_by_version && continue - - # Determine which minor versions to purge - yum_purge_minors=() - yum_keep_minors=() - for m in "${!yum_versions_in_dir[@]}"; do - if [[ "$m" -lt "$TARGET_MINOR" ]]; then - yum_purge_minors+=("$m") - else - yum_keep_minors+=("$m") - fi - done - - # If ALL versions would be purged, keep the highest - yum_highest_purge="" - if [[ ${#yum_keep_minors[@]} -eq 0 && ${#yum_purge_minors[@]} -gt 0 ]]; then - for m in "${yum_purge_minors[@]}"; do - if [[ -z "$yum_highest_purge" || "$m" -gt "$yum_highest_purge" ]]; then - yum_highest_purge="$m" - fi - done - echo " [${vendor}/${os}] No versions >= ${TARGET_MAJOR}.${TARGET_MINOR} found; keeping ${TARGET_MAJOR}.${yum_highest_purge}" - TOTAL_KEPT=$((TOTAL_KEPT + 1)) - fi - - # Remove files for purged versions - for key in "${!yum_files_by_version[@]}"; do - m="${key%%:*}" - f="${key#*:}" - if [[ "$m" -lt "$TARGET_MINOR" && "$m" != "$yum_highest_purge" ]]; then - do_rm "$f" - TOTAL_REMOVED=$((TOTAL_REMOVED + 1)) - fi - done - - unset yum_versions_in_dir yum_files_by_version - done -done -echo "" - -############################################################################### -# Summary -############################################################################### -echo "=== Summary ===" -echo "Items to remove: ${TOTAL_REMOVED}" -if [[ "$TOTAL_KEPT" -gt 0 ]]; then - echo "Repos where newest purge-candidate was kept (no newer versions present): ${TOTAL_KEPT}" -fi -if [[ "$LIVE" -eq 0 ]]; then - echo "" - echo "This was a dry run. Re-run with --live to actually delete." -fi diff --git a/tools/rebuild-apt-repo.sh b/tools/rebuild-apt-repo.sh deleted file mode 100755 index b709e43..0000000 --- a/tools/rebuild-apt-repo.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -if [ $# -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -DISTRO_CODENAME=$1 - -for ARCH in all amd64 i386; do - mkdir -p /var/ftp/pgadmin4/apt/${DISTRO_CODENAME}/dists/pgadmin4/main/binary-${ARCH} -done - -for ARCH in all amd64 i386; do - cd /var/ftp/pgadmin4/apt/${DISTRO_CODENAME} - apt-ftparchive packages -c=/home/pgaupload/aptftp.conf dists/pgadmin4/main/binary-${ARCH} > dists/pgadmin4/main/binary-${ARCH}/Packages || exit 1 - gzip -f -k dists/pgadmin4/main/binary-${ARCH}/Packages || exit 1 -done - -cd /var/ftp/pgadmin4/apt/${DISTRO_CODENAME}/dists/pgadmin4 -apt-ftparchive release -c=/home/pgaupload/aptftp.conf . > Release && gzip -f -k Release || exit 1 - -rm -f Release.gpg -gpg -u packages@pgadmin.org -bao Release.gpg Release || exit 1 - -rm -f InRelease -gpg -u packages@pgadmin.org --clear-sign --output InRelease Release || exit 1 - diff --git a/tools/rebuild-yum-repo.sh b/tools/rebuild-yum-repo.sh deleted file mode 100755 index 0a6ba31..0000000 --- a/tools/rebuild-yum-repo.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -if [ $# -ne 3 ]; then - echo "Usage: $0 " - exit 1 -fi - -DISTRO_FAMILY=$1 -DISTRO_NAME=$2 -DISTRO_VERSION=$3 - -/usr/bin/createrepo_c /var/ftp/pgadmin4/yum/${DISTRO_FAMILY}/${DISTRO_NAME}-${DISTRO_VERSION}-x86_64 || exit 1 -/usr/bin/gpg --yes --detach-sign --armor /var/ftp/pgadmin4/yum/${DISTRO_FAMILY}/${DISTRO_NAME}-${DISTRO_VERSION}-x86_64/repodata/repomd.xml || exit 1 - -cd /var/ftp/pgadmin4/yum/${DISTRO_FAMILY}/ -ln -snf ${DISTRO_NAME}-${DISTRO_VERSION}-x86_64 ${DISTRO_NAME}-${DISTRO_VERSION}Server-x86_64 || exit 1 -ln -snf ${DISTRO_NAME}-${DISTRO_VERSION}-x86_64 ${DISTRO_NAME}-${DISTRO_VERSION}Workstation-x86_64 || exit 1 -ln -snf ${DISTRO_NAME}-${DISTRO_VERSION}-x86_64 ${DISTRO_NAME}-${DISTRO_VERSION}Client-x86_64 || exit 1 diff --git a/tools/sync-ftp-to-s3.py b/tools/sync-ftp-to-s3.py deleted file mode 100755 index dd4dcc6..0000000 --- a/tools/sync-ftp-to-s3.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import json -import os -import shutil -import subprocess -import tempfile - -# Where does the content live? -BUCKET_NAME = 'pgadmin-archive.postgresql.org' - - -# Format file sizes etc -def format_bits(size): - n = 0 - labels = {0: 'b', 1: 'Kb', 2: 'Mb', 3: 'Gb'} - while size > 1000: - size /= 1000 - n += 1 - return '{}{}'.format(round(size, 3), labels[n]) - - -# Create an index.html file -def gen_index(path, dirs, files, output_dir): - os.makedirs(os.path.join(output_dir, path), exist_ok=True) - - index = os.path.join(output_dir, path, 'index.html') - f = open(index, "a") - - f.write('\n') - f.write('\n') - f.write('pgAdmin Download Archive: {}\n'.format(path)) - f.write('\n') - - f.write('\n') - f.write('

pgAdmin Download Archive: {}

\n'.format(path)) - - if len(dirs) > 0: - f.write('

Sub directories

\n') - f.write('
    \n') - - dirs.sort() - for dir in dirs: - if os.path.basename(dir) != 'snapshots' and os.path.basename(dir) != 'redmine': - f.write('
  • {}
    • \n' - .format(dir, os.path.basename(dir))) - - f.write('
\n') - - if len(files) > 0: - f.write('

Files

\n') - f.write('
    \n') - - files.sort() - for file in files: - f.write('
  • {} ({})
    • \n' - .format(file[0], os.path.basename(file[0]), - format_bits(file[1]))) - - f.write('
\n') - - f.write('\n') - f.write('\n') - - f.close() - - -# Get all the objects to process -def get_objects(BUCKET_NAME): - cmd = ['/usr/bin/aws', 's3api', 'list-objects', - '--bucket', BUCKET_NAME, - '--query', 'Contents[].{Key: Key, Size: Size}'] - - result = subprocess.run(cmd, stdout=subprocess.PIPE) - - objects = json.loads(result.stdout) - paths = {} - - for o in objects: - if o['Key'].startswith('redmine'): - continue - - path = os.path.dirname(o['Key']) - file = os.path.basename(o['Key']) - - if file != 'index.html': - if path not in paths: - paths[path] = [] - - if file not in paths[path]: - paths[path].append([file, o['Size']]) - - return paths - - -# Process each path in the data set -def process_paths(paths, output_dir): - # Make sure we have all possible paths in the structure - # At the moment we only have paths that contain files - new = {} - for path in paths: - while path != '': - path = os.path.dirname(path) - if path not in paths and path not in new: - new[path] = {} - - paths.update(new) - - # Get a list of subdirectories for each path - for path in paths: - dirs = [] - - for subdir in paths: - if subdir[:len(path) + 1] == path + '/' or path == '': - base = os.path.basename(subdir) - immediate_child = '/' not in subdir[len(path) + 1:] - if immediate_child and base and base not in dirs: - dirs.append(base) - - gen_index(path, dirs, paths[path], output_dir) - - -# Command line arguments -parser = argparse.ArgumentParser(description='Sync pgAdmin downloads to the archive site.') -parser.add_argument('cf_dist_id', metavar='', nargs=1, - help='the CloudFront distribution ID to invalidate') - -args = parser.parse_args() - -# Temp directory for the output -output_dir = tempfile.mkdtemp(prefix='pgadmin-archive-') - -# Sync content to s3 -sync_cmd = ('/usr/bin/aws s3 sync /usr/local/ftp.pgadmin.org/' - ' s3://pgadmin-archive.postgresql.org/ --exclude "*/snapshots/*"') -os.system(sync_cmd) - -# Get all the objects to process -data = get_objects(BUCKET_NAME) - -# Process each path -process_paths(data, output_dir) - -# Sync indexes to s3 -sync_cmd = '/usr/bin/aws s3 sync {} s3://pgadmin-archive.postgresql.org/'.format(output_dir) -os.system(sync_cmd) - -# Invalidate CloudFront -sync_cmd = ('/usr/bin/aws cloudfront create-invalidation' - ' --distribution-id {} --paths "/*"').format(args.cf_dist_id[0]) -os.system(sync_cmd) - -# Be a little careful here -if output_dir[:5] == '/tmp/' and len(output_dir) > 6: - shutil.rmtree(output_dir)