diff --git a/AUTHORS b/AUTHORS index 267d9bc57..27ac11ee7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -89,3 +89,4 @@ List of contributors, in chronological order: * James Munson (https://github.com/jmunson) * Luan Dang (https://github.com/luanmdang) * Martin Simon (https://github.com/barnumbirr) +* somaz (https://github.com/somaz94) diff --git a/deb/package_files.go b/deb/package_files.go index 2a805d83a..337ef7b9e 100644 --- a/deb/package_files.go +++ b/deb/package_files.go @@ -48,6 +48,10 @@ func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error) return f.PoolPath, err } +func (f *PackageFile) isSHA512Only() bool { + return f.Checksums.MD5 == "" && f.Checksums.SHA256 == "" && f.Checksums.SHA512 != "" +} + // DownloadURL return relative URL to package download location func (f *PackageFile) DownloadURL() string { return filepath.Join(f.downloadPath, f.Filename) diff --git a/deb/remote.go b/deb/remote.go index 92274afa8..1654096f8 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -718,6 +718,63 @@ func (repo *RemoteRepo) FilterLatest(count int) (oldLen, newLen int, err error) return } +// restoreSHA512OnlyFiles reuses the previous update's pool path and checksums +// for files indexed by SHA512 alone, whose pool path cannot be derived. +func (repo *RemoteRepo) restoreSHA512OnlyFiles(p *Package, packageCollection *PackageCollection) error { + if repo.packageRefs == nil { + return nil + } + + files := p.Files() + sha512Only := false + for _, f := range files { + if f.isSHA512Only() { + sha512Only = true + break + } + } + if !sha512Only { + return nil + } + + // packageRefs is built from a list unique by ShortKey, so at most one ref matches. + prefix := append(p.ShortKey(""), ' ') + refs := repo.packageRefs.Refs + i := sort.Search(len(refs), func(j int) bool { return bytes.Compare(refs[j], prefix) >= 0 }) + if i == len(refs) || !bytes.HasPrefix(refs[i], prefix) { + return nil + } + + prevP, err := packageCollection.ByKey(refs[i]) + if errors.Is(err, database.ErrNotFound) { + return nil + } + if err != nil { + return err + } + + restored := false + for idx := range files { + f := &files[idx] + if !f.isSHA512Only() { + continue + } + for _, prevF := range prevP.Files() { + if prevF.Filename == f.Filename && prevF.Checksums.Size == f.Checksums.Size && prevF.Checksums.SHA512 == f.Checksums.SHA512 { + f.Checksums = prevF.Checksums + f.PoolPath = prevF.PoolPath + restored = true + break + } + } + } + + if restored { + p.UpdateFiles(files) + } + return nil +} + // BuildDownloadQueue builds queue, discards current PackageList func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) { if repo.packageList == nil { @@ -729,6 +786,10 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packag seen := make(map[string]int, repo.packageList.Len()) err = repo.packageList.ForEach(func(p *Package) error { + if err2 := repo.restoreSHA512OnlyFiles(p, packageCollection); err2 != nil { + return err2 + } + if repo.packageRefs != nil && skipExistingPackages { if repo.packageRefs.Has(p) { // skip this package, but load checksums/files from package in DB diff --git a/deb/remote_test.go b/deb/remote_test.go index f154d1d0c..cace40141 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -7,6 +7,7 @@ import ( "io" "os" "sort" + "strings" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/console" @@ -731,6 +732,50 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { c.Assert(s.flat.packageRefs, NotNil) } +func (s *RemoteRepoSuite) TestDownloadSHA512OnlyFlat(c *C) { + downloader := http.NewFakeDownloader() + fetchIndexes := func(packages string) { + downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404}) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404}) + downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404}) + downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", packages) + + c.Assert(s.flat.Fetch(downloader, nil, true), IsNil) + c.Assert(s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, true), IsNil) + c.Assert(downloader.Empty(), Equals, true) + } + + fetchIndexes(exampleSHA512OnlyPackagesFile) + queue, _, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + c.Assert(err, IsNil) + c.Assert(queue, HasLen, 1) + + tmpPath := c.MkDir() + "/amanda-client.deb" + c.Assert(os.WriteFile(tmpPath, []byte("abc"), 0644), IsNil) + queue[0].File.PoolPath, err = s.packagePool.Import(tmpPath, queue[0].File.Filename, &queue[0].File.Checksums, true, s.cs) + c.Assert(err, IsNil) + c.Assert(s.flat.FinalizeDownload(s.collectionFactory, nil), IsNil) + + for _, skipExisting := range []bool{true, false} { + fetchIndexes(exampleSHA512OnlyPackagesFile) + queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, skipExisting) + c.Assert(err, IsNil) + c.Check(size, Equals, int64(0)) + c.Check(queue, HasLen, 0) + c.Check(s.flat.packageList.ForEach(func(p *Package) error { + c.Check(s.flat.packageRefs.Has(p), Equals, true) + return nil + }), IsNil) + c.Assert(s.flat.FinalizeDownload(s.collectionFactory, nil), IsNil) + } + + fetchIndexes(strings.Replace(exampleSHA512OnlyPackagesFile, "SHA512: ddaf35", "SHA512: 00af35", 1)) + queue, _, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false) + c.Assert(err, IsNil) + c.Check(queue, HasLen, 1) +} + func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { s.flat.DownloadSources = true @@ -1174,6 +1219,16 @@ SHA1: 66b27417d37e024c46526c2f6d358a754fc552f3 SHA256: 3608bca1e44ea6c4d268eb6db02260269892c0b42b86bbf1e77a6fa16c3c9282 ` +const exampleSHA512OnlyPackagesFile = `Package: amanda-client +Source: amanda +Version: 1:3.3.1-3~bpo60+1 +Architecture: i386 +Description: Advanced Maryland Automatic Network Disk Archiver (Client) +Filename: pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb +Size: 3 +SHA512: ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f +` + const exampleInstallerHashSumFile = `82f69d557f0004d2923fb03e4fb47d18187e37768dbfd0c99756f8a6c68a6d3a ./MANIFEST `