mirror: stop re-downloading packages indexed by SHA512 only

AUTHORS

@@ -89,3 +89,4 @@ List of contributors, in chronological order:
 * James Munson (https://github.com/jmunson)
 * Luan Dang (https://github.com/luanmdang)
 * Martin Simon (https://github.com/barnumbirr)
+* somaz (https://github.com/somaz94)

deb/package_files.go

@@ -48,6 +48,10 @@ func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error)
 	return f.PoolPath, err
 }
 
+func (f *PackageFile) isSHA512Only() bool {
+	return f.Checksums.MD5 == "" && f.Checksums.SHA256 == "" && f.Checksums.SHA512 != ""
+}
+
 // DownloadURL return relative URL to package download location
 func (f *PackageFile) DownloadURL() string {
 	return filepath.Join(f.downloadPath, f.Filename)

deb/remote.go

@@ -718,6 +718,63 @@ func (repo *RemoteRepo) FilterLatest(count int) (oldLen, newLen int, err error)
 	return
 }
 
+// restoreSHA512OnlyFiles reuses the previous update's pool path and checksums
+// for files indexed by SHA512 alone, whose pool path cannot be derived.
+func (repo *RemoteRepo) restoreSHA512OnlyFiles(p *Package, packageCollection *PackageCollection) error {
+	if repo.packageRefs == nil {
+		return nil
+	}
+
+	files := p.Files()
+	sha512Only := false
+	for _, f := range files {
+		if f.isSHA512Only() {
+			sha512Only = true
+			break
+		}
+	}
+	if !sha512Only {
+		return nil
+	}
+
+	// packageRefs is built from a list unique by ShortKey, so at most one ref matches.
+	prefix := append(p.ShortKey(""), ' ')
+	refs := repo.packageRefs.Refs
+	i := sort.Search(len(refs), func(j int) bool { return bytes.Compare(refs[j], prefix) >= 0 })
+	if i == len(refs) || !bytes.HasPrefix(refs[i], prefix) {
+		return nil
+	}
+
+	prevP, err := packageCollection.ByKey(refs[i])
+	if errors.Is(err, database.ErrNotFound) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	restored := false
+	for idx := range files {
+		f := &files[idx]
+		if !f.isSHA512Only() {
+			continue
+		}
+		for _, prevF := range prevP.Files() {
+			if prevF.Filename == f.Filename && prevF.Checksums.Size == f.Checksums.Size && prevF.Checksums.SHA512 == f.Checksums.SHA512 {
+				f.Checksums = prevF.Checksums
+				f.PoolPath = prevF.PoolPath
+				restored = true
+				break
+			}
+		}
+	}
+
+	if restored {
+		p.UpdateFiles(files)
+	}
+	return nil
+}
+
 // BuildDownloadQueue builds queue, discards current PackageList
 func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
 	if repo.packageList == nil {
@@ -729,6 +786,10 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packag
 	seen := make(map[string]int, repo.packageList.Len())
 
 	err = repo.packageList.ForEach(func(p *Package) error {
+		if err2 := repo.restoreSHA512OnlyFiles(p, packageCollection); err2 != nil {
+			return err2
+		}
+
 		if repo.packageRefs != nil && skipExistingPackages {
 			if repo.packageRefs.Has(p) {
 				// skip this package, but load checksums/files from package in DB

deb/remote_test.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"sort"
+	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/console"
@@ -731,6 +732,50 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(s.flat.packageRefs, NotNil)
 }
 
+func (s *RemoteRepoSuite) TestDownloadSHA512OnlyFlat(c *C) {
+	downloader := http.NewFakeDownloader()
+	fetchIndexes := func(packages string) {
+		downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+		downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+		downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+		downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+		downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", packages)
+
+		c.Assert(s.flat.Fetch(downloader, nil, true), IsNil)
+		c.Assert(s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, true), IsNil)
+		c.Assert(downloader.Empty(), Equals, true)
+	}
+
+	fetchIndexes(exampleSHA512OnlyPackagesFile)
+	queue, _, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Assert(queue, HasLen, 1)
+
+	tmpPath := c.MkDir() + "/amanda-client.deb"
+	c.Assert(os.WriteFile(tmpPath, []byte("abc"), 0644), IsNil)
+	queue[0].File.PoolPath, err = s.packagePool.Import(tmpPath, queue[0].File.Filename, &queue[0].File.Checksums, true, s.cs)
+	c.Assert(err, IsNil)
+	c.Assert(s.flat.FinalizeDownload(s.collectionFactory, nil), IsNil)
+
+	for _, skipExisting := range []bool{true, false} {
+		fetchIndexes(exampleSHA512OnlyPackagesFile)
+		queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, skipExisting)
+		c.Assert(err, IsNil)
+		c.Check(size, Equals, int64(0))
+		c.Check(queue, HasLen, 0)
+		c.Check(s.flat.packageList.ForEach(func(p *Package) error {
+			c.Check(s.flat.packageRefs.Has(p), Equals, true)
+			return nil
+		}), IsNil)
+		c.Assert(s.flat.FinalizeDownload(s.collectionFactory, nil), IsNil)
+	}
+
+	fetchIndexes(strings.Replace(exampleSHA512OnlyPackagesFile, "SHA512: ddaf35", "SHA512: 00af35", 1))
+	queue, _, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(queue, HasLen, 1)
+}
+
 func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	s.flat.DownloadSources = true
 
@@ -1174,6 +1219,16 @@ SHA1: 66b27417d37e024c46526c2f6d358a754fc552f3
 SHA256: 3608bca1e44ea6c4d268eb6db02260269892c0b42b86bbf1e77a6fa16c3c9282
 `
 
+const exampleSHA512OnlyPackagesFile = `Package: amanda-client
+Source: amanda
+Version: 1:3.3.1-3~bpo60+1
+Architecture: i386
+Description: Advanced Maryland Automatic Network Disk Archiver (Client)
+Filename: pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb
+Size: 3
+SHA512: ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f
+`
+
 const exampleInstallerHashSumFile = `82f69d557f0004d2923fb03e4fb47d18187e37768dbfd0c99756f8a6c68a6d3a  ./MANIFEST
 `