diff --git a/tools/README.md b/tools/README.md index 5ef9638..f4605e7 100644 --- a/tools/README.md +++ b/tools/README.md @@ -4,50 +4,30 @@ This directory contains various scripts used to manage www.pgadmin.org and pgAdm **Most are very system-specific; don't expect them to work for you!!** -**Note:** The AWS CLI is expected to be installed at /usr/bin/aws. +On the production web server, all scripts **must** be run as the *pgaupload* user. -Om the production web server, all scripts **must** be run as the *pgaupload* user. +## create_release.py -## create-release.py - -This script is called by the Jenkins server as part of the release process. It creates all the required objects in the -website database for a new release. +This script is called as part of the release process, through the `create-release` verb of the publishing wrapper +(`pkg/publish/pga-publish` in the pgadmin4 repo) which the GitHub Actions workflows reach over SSH. It creates all the +required objects in the website database for a new release. ## load-docs.sh -This script is called by the Jenkins server as part of the release process. It unpacks the docs for the version of -pgAdmin specified on the command line, and loads them into the website database. - -## purge-cache.sh - -This script is called by the Jenkins server as part of the release process. It purges the entire Varnish cache that -sits in front of Nginx. - -# rebuild-apt-repo.sh +This script is called as part of the release process, through the `load-docs` verb of the publishing wrapper. It +unpacks the docs for the version of pgAdmin specified on the command line, and loads them into the website database. -This script will rebuild the specified APT repo metadata, and re-sign everything. This is typically used if old -packages are manually removed. +Because the tarball it unpacks is built elsewhere and lands in a tree that an upload key can write to, the script +treats it as untrusted input and vets the member list before unpacking anything; the comments in the script explain +what each check is there to stop. -```bash -./rebuild-apt-repo.sh focal -``` - -# rebuild-yum-repo.sh - -This script will rebuild the specified YUM repo metadata, and re-sign everything. This is typically used if old -packages are manually removed. - -```bash -./rebuild-yum-repo.sh redhat rhel 8 -``` - -## sync-ftp-to-s3.py +## purge-cache.sh -This script is called by the Jenkins server as part of the release process. It will copy all the packages to the S3 -bucket that hosts https://pgadmin-archive.postgresql.org, and generate and sync HTML index pages for browsing. +This script is called as part of the release process, through the `purge-cache` verb of the publishing wrapper. It +purges the entire Varnish cache that sits in front of Nginx. ## update-docs.sh -This script is typically called from cron. It updates the git checkout of pgAdmin at -/var/www/pgaweb/static/docs/pgadmin4-dev, and then rebuilds the HTML docs which are subsequently loaded into the -website database as the 'dev' docs. +This script is run from cron on the web server, twice an hour, as the *pgaupload* user, and is not part of the release +process. It updates the git checkout of pgAdmin at /var/www/pgaweb/static/docs/pgadmin4-dev, and then rebuilds the HTML +docs which are subsequently loaded into the website database as the 'dev' docs. diff --git a/tools/load-docs.sh b/tools/load-docs.sh index bc28157..492cd15 100755 --- a/tools/load-docs.sh +++ b/tools/load-docs.sh @@ -1,27 +1,127 @@ #!/bin/bash +set -e +set -o pipefail + +DOCS_ROOT=/var/www/pgaweb/static/docs +FTP_ROOT=/var/ftp/pgadmin4 + if [ $# -ne 1 ] then echo "usage: $0 " exit 1 fi +VERSION=$1 + +# The version is interpolated into the tarball path and into the name of the +# directory we unpack into, so its shape is checked before it is used for +# either, because a value carrying a slash, a '..' or a shell metacharacter +# would let whoever called us choose those paths rather than merely name a +# release. +if [[ ! ${VERSION} =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] +then + echo "Invalid version number (${VERSION}); expected something like 9.18 or 9.18.1." + exit 1 +fi + +TARBALL=${FTP_ROOT}/v${VERSION}/docs/pgadmin4-${VERSION}-docs.tar.gz +DOCS_DIR=pgadmin4-${VERSION}-docs + # Check the docs file exists -TARBALL=/var/ftp/pgadmin4/v${1}/docs/pgadmin4-${1}-docs.tar.gz -if [ ! -f ${TARBALL} ] +if [ ! -f "${TARBALL}" ] then echo "The source tarball (${TARBALL}) could not be found." exit 1 fi +# Everything below happens in a staging directory beside the live one, so that +# a tarball which turns out to be bad, or an unpacking that dies part way +# through, cannot leave the site serving a half replaced tree. It sits inside +# DOCS_ROOT so that moving the result into place is a rename rather than a copy +# of the best part of a gigabyte. +STAGING=$(mktemp -d "${DOCS_ROOT}/.load-docs.XXXXXX") +trap 'rm -rf "${STAGING}"' EXIT + +# The tarball is assembled by a CI runner and lands in a tree that an upload +# key can write to, so it is untrusted input, and an archive decides for itself +# where its contents are written. We therefore vet the member list before +# unpacking anything, rather than trusting whatever a given tar happens to do +# with a hostile archive, since that varies between implementations and +# versions and is not a control we own. The two listings also fail the script +# outright if the archive cannot be read from end to end. +tar -tzvf "${TARBALL}" > "${STAGING}/listing.verbose" +tar -tzf "${TARBALL}" > "${STAGING}/listing.names" + +# Every member must be a plain file or a directory, which rejects symlinks and +# hard links whose targets could redirect a later member out of the docs tree, +# along with devices and anything else that has no business in a docs tarball. +# A listing line we cannot make sense of fails this test as well, which is the +# right way round for it to fail. +if cut -c1 "${STAGING}/listing.verbose" | sort -u | grep -qv '^[-d]$' +then + echo "${TARBALL} holds members that are neither plain files nor directories." + exit 1 +fi + +# Every member must also sit under the one directory we expect, which rejects +# absolute paths and '..' traversal, and confirms the archive carries the +# single top level directory that the docloader below is pointed at. +while read -r MEMBER +do + case ${MEMBER} in + "${DOCS_DIR}"|"${DOCS_DIR}"/) + ;; + "${DOCS_DIR}"/*) + case ${MEMBER} in + */../*|*/..) + echo "${TARBALL} holds a member traversing out of ${DOCS_DIR} (${MEMBER})." + exit 1 + ;; + esac + ;; + *) + echo "${TARBALL} holds a member outside ${DOCS_DIR} (${MEMBER})." + exit 1 + ;; + esac +done < "${STAGING}/listing.names" + +rm -f "${STAGING}/listing.verbose" "${STAGING}/listing.names" + # Unpack the tarball -cd /var/www/pgaweb/static/docs -tar -zxvf ${TARBALL} +tar -xzf "${TARBALL}" -C "${STAGING}" --no-same-owner --no-same-permissions + +# Confirm we got out what we expected, rather than assuming that a tar which +# exited zero wrote the docs we are about to load and publish. The second check +# is belt and braces over the member vetting above, and catches a tar that +# creates something odd from an archive whose listing looked ordinary. +if [ ! -f "${STAGING}/${DOCS_DIR}/index.html" ] +then + echo "${TARBALL} did not unpack to ${DOCS_DIR} holding an index.html." + exit 1 +fi + +if [ -n "$(find "${STAGING}" ! -type d ! -type f -print -quit)" ] +then + echo "${TARBALL} unpacked something that is neither a plain file nor a directory." + exit 1 +fi + +# Swap the new docs into place, keeping the old copy until the new one has +# landed so that reloading a version we already hold does not leave a gap. +if [ -e "${DOCS_ROOT}/${DOCS_DIR}" ] +then + rm -rf "${DOCS_ROOT}/${DOCS_DIR}.replaced" + mv "${DOCS_ROOT}/${DOCS_DIR}" "${DOCS_ROOT}/${DOCS_DIR}.replaced" +fi +mv "${STAGING}/${DOCS_DIR}" "${DOCS_ROOT}/${DOCS_DIR}" +rm -rf "${DOCS_ROOT}/${DOCS_DIR}.replaced" # Load 'em cd /var/www/pgaweb source /usr/share/python3/pginfra-virtualenv-django52-py3/bin/activate -./manage.py docloader pgadmin4 ${1} pgadmin4-${1}-docs +./manage.py docloader pgadmin4 "${VERSION}" "${DOCS_DIR}" # Clear the cache sudo varnishadm "ban req.url ~ ." diff --git a/tools/purge_pgadmin_releases.sh b/tools/purge_pgadmin_releases.sh deleted file mode 100755 index 5605cc3..0000000 --- a/tools/purge_pgadmin_releases.sh +++ /dev/null @@ -1,266 +0,0 @@ -#!/bin/bash -# -# purge_pgadmin_releases.sh - Purge old pgAdmin releases -# -# Removes old pgAdmin version directories, APT packages, and YUM packages -# for a given major version, keeping only the specified version and later. -# -# Usage: purge_pgadmin_releases.sh [--live] -# -# Example: purge_pgadmin_releases.sh 9.11 -# This will purge versions 9.0 through 9.10 but NOT 8.x, 7.x, etc. -# -# By default runs in dry-run mode. Use --live to actually delete files. -# -# If an OS/distro repo directory only contains pgAdmin versions that would -# be purged, the most recent of those versions is kept so the repo is not -# left empty of that major version. - -set -euo pipefail - -BASE_DIR="/var/ftp/pgadmin4" -LIVE=0 - -usage() { - echo "Usage: $0 [--live] " - echo "" - echo " Purges all pgAdmin releases older than within the" - echo " same major version." - echo "" - echo " --live Actually delete files (default is dry-run)" - echo "" - echo " Example: $0 9.11" - echo " Purges 9.0 through 9.10, keeps 9.11 and later." - exit 1 -} - -# Parse arguments -while [[ $# -gt 0 ]]; do - case "$1" in - --live) - LIVE=1 - shift - ;; - -h|--help) - usage - ;; - -*) - echo "Error: Unknown option: $1" - usage - ;; - *) - VERSION="$1" - shift - ;; - esac -done - -if [[ -z "${VERSION:-}" ]]; then - echo "Error: Version argument is required." - usage -fi - -# Validate version format -if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+$'; then - echo "Error: Version must be in major.minor format (e.g. 9.11)" - exit 1 -fi - -TARGET_MAJOR="${VERSION%%.*}" -TARGET_MINOR="${VERSION#*.}" - -if [[ "$LIVE" -eq 1 ]]; then - echo "*** LIVE MODE - files will be deleted ***" -else - echo "*** DRY RUN MODE - no files will be deleted (use --live to delete) ***" -fi -echo "" -echo "Purging pgAdmin ${TARGET_MAJOR}.0 through ${TARGET_MAJOR}.$((TARGET_MINOR - 1))" -echo "" - -# Helper: execute or print an rm command -do_rm() { - if [[ "$LIVE" -eq 1 ]]; then - rm -rf "$@" - else - echo " Would remove: $*" - fi -} - -# Track totals -TOTAL_REMOVED=0 -TOTAL_KEPT=0 - -############################################################################### -# 1. Purge version directories (e.g. /var/ftp/pgadmin4/v9.0) -############################################################################### -echo "=== Version directories ===" -for minor in $(seq 0 $((TARGET_MINOR - 1))); do - VDIR="${BASE_DIR}/v${TARGET_MAJOR}.${minor}" - if [[ -d "$VDIR" ]]; then - do_rm "$VDIR" - TOTAL_REMOVED=$((TOTAL_REMOVED + 1)) - fi -done -echo "" - -############################################################################### -# 2. Purge APT packages -############################################################################### -echo "=== APT repositories ===" -for distro_dir in "${BASE_DIR}"/apt/*/; do - [[ -d "$distro_dir" ]] || continue - distro=$(basename "$distro_dir") - - # Find all binary directories under dists/pgadmin4/main/ - for bin_dir in "${distro_dir}"dists/pgadmin4/main/binary-*/; do - [[ -d "$bin_dir" ]] || continue - arch=$(basename "$bin_dir") - - # Collect pgadmin4 .deb files for the target major version in this dir - # File patterns: pgadmin4_{ver}_all.deb, pgadmin4-desktop_{ver}_amd64.deb, etc. - # Version in filename can be like "9.5" or "9.10-1.bookworm" - declare -A versions_in_dir=() - declare -A files_by_version=() - - while IFS= read -r -d '' deb_file; do - fname=$(basename "$deb_file") - # Extract version: pgadmin4[-something]_{VERSION}_{arch}.deb - # VERSION is like "9.5" or "9.10-1.bookworm" - pkg_ver=$(echo "$fname" | sed -E 's/^pgadmin4[^_]*_([0-9]+\.[0-9]+).*/\1/') - pkg_major="${pkg_ver%%.*}" - pkg_minor="${pkg_ver#*.}" - - if [[ "$pkg_major" == "$TARGET_MAJOR" ]]; then - versions_in_dir["$pkg_minor"]=1 - files_by_version["${pkg_minor}:${deb_file}"]="$deb_file" - fi - done < <(find "$bin_dir" -maxdepth 1 -name "pgadmin4*.deb" -print0 2>/dev/null) - - [[ ${#versions_in_dir[@]} -eq 0 ]] && unset versions_in_dir files_by_version && continue - - # Determine which minor versions to purge (< TARGET_MINOR) - purge_minors=() - keep_minors=() - for m in "${!versions_in_dir[@]}"; do - if [[ "$m" -lt "$TARGET_MINOR" ]]; then - purge_minors+=("$m") - else - keep_minors+=("$m") - fi - done - - # If ALL versions of this major would be purged, keep the highest - highest_purge="" - if [[ ${#keep_minors[@]} -eq 0 && ${#purge_minors[@]} -gt 0 ]]; then - for m in "${purge_minors[@]}"; do - if [[ -z "$highest_purge" || "$m" -gt "$highest_purge" ]]; then - highest_purge="$m" - fi - done - echo " [${distro}/${arch}] No versions >= ${TARGET_MAJOR}.${TARGET_MINOR} found; keeping ${TARGET_MAJOR}.${highest_purge}" - TOTAL_KEPT=$((TOTAL_KEPT + 1)) - fi - - # Remove files for purged versions - for key in "${!files_by_version[@]}"; do - m="${key%%:*}" - f="${key#*:}" - if [[ "$m" -lt "$TARGET_MINOR" && "$m" != "$highest_purge" ]]; then - do_rm "$f" - TOTAL_REMOVED=$((TOTAL_REMOVED + 1)) - fi - done - - unset versions_in_dir files_by_version - done -done -echo "" - -############################################################################### -# 3. Purge YUM/DNF packages -############################################################################### -echo "=== YUM repositories ===" -for vendor_dir in "${BASE_DIR}"/yum/*/; do - [[ -d "$vendor_dir" ]] || continue - vendor=$(basename "$vendor_dir") - - # Skip non-directory files (like the .noarch.rpm and README at the top level) - for os_dir in "${vendor_dir}"*/; do - [[ -d "$os_dir" ]] || continue - # Skip symlinks (e.g. fedora-41Client-x86_64 -> fedora-41-x86_64) - [[ -L "${os_dir%/}" ]] && continue - os=$(basename "$os_dir") - - # Skip repodata directories and non-repo items - [[ "$os" == "repodata" ]] && continue - - # Collect pgadmin4 RPMs for the target major version - declare -A yum_versions_in_dir=() - declare -A yum_files_by_version=() - - while IFS= read -r -d '' rpm_file; do - fname=$(basename "$rpm_file") - # Extract version: pgadmin4[-something]-{MAJOR}.{MINOR}-{release}.{dist}.{arch}.rpm - pkg_ver=$(echo "$fname" | sed -E 's/.*pgadmin4[^0-9]*([0-9]+\.[0-9]+)-.*/\1/') - pkg_major="${pkg_ver%%.*}" - pkg_minor="${pkg_ver#*.}" - - if [[ "$pkg_major" == "$TARGET_MAJOR" ]]; then - yum_versions_in_dir["$pkg_minor"]=1 - yum_files_by_version["${pkg_minor}:${rpm_file}"]="$rpm_file" - fi - done < <(find "$os_dir" -maxdepth 1 -name "pgadmin4*.rpm" -print0 2>/dev/null) - - [[ ${#yum_versions_in_dir[@]} -eq 0 ]] && unset yum_versions_in_dir yum_files_by_version && continue - - # Determine which minor versions to purge - yum_purge_minors=() - yum_keep_minors=() - for m in "${!yum_versions_in_dir[@]}"; do - if [[ "$m" -lt "$TARGET_MINOR" ]]; then - yum_purge_minors+=("$m") - else - yum_keep_minors+=("$m") - fi - done - - # If ALL versions would be purged, keep the highest - yum_highest_purge="" - if [[ ${#yum_keep_minors[@]} -eq 0 && ${#yum_purge_minors[@]} -gt 0 ]]; then - for m in "${yum_purge_minors[@]}"; do - if [[ -z "$yum_highest_purge" || "$m" -gt "$yum_highest_purge" ]]; then - yum_highest_purge="$m" - fi - done - echo " [${vendor}/${os}] No versions >= ${TARGET_MAJOR}.${TARGET_MINOR} found; keeping ${TARGET_MAJOR}.${yum_highest_purge}" - TOTAL_KEPT=$((TOTAL_KEPT + 1)) - fi - - # Remove files for purged versions - for key in "${!yum_files_by_version[@]}"; do - m="${key%%:*}" - f="${key#*:}" - if [[ "$m" -lt "$TARGET_MINOR" && "$m" != "$yum_highest_purge" ]]; then - do_rm "$f" - TOTAL_REMOVED=$((TOTAL_REMOVED + 1)) - fi - done - - unset yum_versions_in_dir yum_files_by_version - done -done -echo "" - -############################################################################### -# Summary -############################################################################### -echo "=== Summary ===" -echo "Items to remove: ${TOTAL_REMOVED}" -if [[ "$TOTAL_KEPT" -gt 0 ]]; then - echo "Repos where newest purge-candidate was kept (no newer versions present): ${TOTAL_KEPT}" -fi -if [[ "$LIVE" -eq 0 ]]; then - echo "" - echo "This was a dry run. Re-run with --live to actually delete." -fi diff --git a/tools/rebuild-apt-repo.sh b/tools/rebuild-apt-repo.sh deleted file mode 100755 index b709e43..0000000 --- a/tools/rebuild-apt-repo.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -if [ $# -ne 1 ]; then - echo "Usage: $0 " - exit 1 -fi - -DISTRO_CODENAME=$1 - -for ARCH in all amd64 i386; do - mkdir -p /var/ftp/pgadmin4/apt/${DISTRO_CODENAME}/dists/pgadmin4/main/binary-${ARCH} -done - -for ARCH in all amd64 i386; do - cd /var/ftp/pgadmin4/apt/${DISTRO_CODENAME} - apt-ftparchive packages -c=/home/pgaupload/aptftp.conf dists/pgadmin4/main/binary-${ARCH} > dists/pgadmin4/main/binary-${ARCH}/Packages || exit 1 - gzip -f -k dists/pgadmin4/main/binary-${ARCH}/Packages || exit 1 -done - -cd /var/ftp/pgadmin4/apt/${DISTRO_CODENAME}/dists/pgadmin4 -apt-ftparchive release -c=/home/pgaupload/aptftp.conf . > Release && gzip -f -k Release || exit 1 - -rm -f Release.gpg -gpg -u packages@pgadmin.org -bao Release.gpg Release || exit 1 - -rm -f InRelease -gpg -u packages@pgadmin.org --clear-sign --output InRelease Release || exit 1 - diff --git a/tools/rebuild-yum-repo.sh b/tools/rebuild-yum-repo.sh deleted file mode 100755 index 0a6ba31..0000000 --- a/tools/rebuild-yum-repo.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -if [ $# -ne 3 ]; then - echo "Usage: $0 " - exit 1 -fi - -DISTRO_FAMILY=$1 -DISTRO_NAME=$2 -DISTRO_VERSION=$3 - -/usr/bin/createrepo_c /var/ftp/pgadmin4/yum/${DISTRO_FAMILY}/${DISTRO_NAME}-${DISTRO_VERSION}-x86_64 || exit 1 -/usr/bin/gpg --yes --detach-sign --armor /var/ftp/pgadmin4/yum/${DISTRO_FAMILY}/${DISTRO_NAME}-${DISTRO_VERSION}-x86_64/repodata/repomd.xml || exit 1 - -cd /var/ftp/pgadmin4/yum/${DISTRO_FAMILY}/ -ln -snf ${DISTRO_NAME}-${DISTRO_VERSION}-x86_64 ${DISTRO_NAME}-${DISTRO_VERSION}Server-x86_64 || exit 1 -ln -snf ${DISTRO_NAME}-${DISTRO_VERSION}-x86_64 ${DISTRO_NAME}-${DISTRO_VERSION}Workstation-x86_64 || exit 1 -ln -snf ${DISTRO_NAME}-${DISTRO_VERSION}-x86_64 ${DISTRO_NAME}-${DISTRO_VERSION}Client-x86_64 || exit 1 diff --git a/tools/sync-ftp-to-s3.py b/tools/sync-ftp-to-s3.py deleted file mode 100755 index dd4dcc6..0000000 --- a/tools/sync-ftp-to-s3.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import json -import os -import shutil -import subprocess -import tempfile - -# Where does the content live? -BUCKET_NAME = 'pgadmin-archive.postgresql.org' - - -# Format file sizes etc -def format_bits(size): - n = 0 - labels = {0: 'b', 1: 'Kb', 2: 'Mb', 3: 'Gb'} - while size > 1000: - size /= 1000 - n += 1 - return '{}{}'.format(round(size, 3), labels[n]) - - -# Create an index.html file -def gen_index(path, dirs, files, output_dir): - os.makedirs(os.path.join(output_dir, path), exist_ok=True) - - index = os.path.join(output_dir, path, 'index.html') - f = open(index, "a") - - f.write('\n') - f.write('\n') - f.write('pgAdmin Download Archive: {}\n'.format(path)) - f.write('\n') - - f.write('\n') - f.write('

pgAdmin Download Archive: {}

\n'.format(path)) - - if len(dirs) > 0: - f.write('

Sub directories

\n') - f.write('
    \n') - - dirs.sort() - for dir in dirs: - if os.path.basename(dir) != 'snapshots' and os.path.basename(dir) != 'redmine': - f.write('
  • {}
    • \n' - .format(dir, os.path.basename(dir))) - - f.write('
\n') - - if len(files) > 0: - f.write('

Files

\n') - f.write('
    \n') - - files.sort() - for file in files: - f.write('
  • {} ({})
    • \n' - .format(file[0], os.path.basename(file[0]), - format_bits(file[1]))) - - f.write('
\n') - - f.write('\n') - f.write('\n') - - f.close() - - -# Get all the objects to process -def get_objects(BUCKET_NAME): - cmd = ['/usr/bin/aws', 's3api', 'list-objects', - '--bucket', BUCKET_NAME, - '--query', 'Contents[].{Key: Key, Size: Size}'] - - result = subprocess.run(cmd, stdout=subprocess.PIPE) - - objects = json.loads(result.stdout) - paths = {} - - for o in objects: - if o['Key'].startswith('redmine'): - continue - - path = os.path.dirname(o['Key']) - file = os.path.basename(o['Key']) - - if file != 'index.html': - if path not in paths: - paths[path] = [] - - if file not in paths[path]: - paths[path].append([file, o['Size']]) - - return paths - - -# Process each path in the data set -def process_paths(paths, output_dir): - # Make sure we have all possible paths in the structure - # At the moment we only have paths that contain files - new = {} - for path in paths: - while path != '': - path = os.path.dirname(path) - if path not in paths and path not in new: - new[path] = {} - - paths.update(new) - - # Get a list of subdirectories for each path - for path in paths: - dirs = [] - - for subdir in paths: - if subdir[:len(path) + 1] == path + '/' or path == '': - base = os.path.basename(subdir) - immediate_child = '/' not in subdir[len(path) + 1:] - if immediate_child and base and base not in dirs: - dirs.append(base) - - gen_index(path, dirs, paths[path], output_dir) - - -# Command line arguments -parser = argparse.ArgumentParser(description='Sync pgAdmin downloads to the archive site.') -parser.add_argument('cf_dist_id', metavar='', nargs=1, - help='the CloudFront distribution ID to invalidate') - -args = parser.parse_args() - -# Temp directory for the output -output_dir = tempfile.mkdtemp(prefix='pgadmin-archive-') - -# Sync content to s3 -sync_cmd = ('/usr/bin/aws s3 sync /usr/local/ftp.pgadmin.org/' - ' s3://pgadmin-archive.postgresql.org/ --exclude "*/snapshots/*"') -os.system(sync_cmd) - -# Get all the objects to process -data = get_objects(BUCKET_NAME) - -# Process each path -process_paths(data, output_dir) - -# Sync indexes to s3 -sync_cmd = '/usr/bin/aws s3 sync {} s3://pgadmin-archive.postgresql.org/'.format(output_dir) -os.system(sync_cmd) - -# Invalidate CloudFront -sync_cmd = ('/usr/bin/aws cloudfront create-invalidation' - ' --distribution-id {} --paths "/*"').format(args.cf_dist_id[0]) -os.system(sync_cmd) - -# Be a little careful here -if output_dir[:5] == '/tmp/' and len(output_dir) > 6: - shutil.rmtree(output_dir)