diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 6f4783b88..2f6a18bba 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -20,6 +20,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/source" "github.com/canonical/chisel/internal/tarball" ) @@ -90,7 +91,7 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) + pkgSources, err := source.Resolve(options.Archives, options.Selection) if err != nil { return err } @@ -108,7 +109,7 @@ func Run(options *RunOptions) error { extractPackage = make(map[string][]tarball.ExtractInfo) extract[slice.Package] = extractPackage } - arch := pkgArchive[slice.Package].Options().Arch + arch := pkgSources[slice.Package].Arch() for targetPath, pathInfo := range slice.Contents { if targetPath == "" { continue @@ -153,7 +154,7 @@ func Run(options *RunOptions) error { continue } pkg := options.Selection.Release.Packages[slice.Package] - reader, info, err := pkgArchive[slice.Package].Fetch(pkg.RealName) + reader, info, err := pkgSources[pkg.Name].Fetch() if err != nil { return err } @@ -270,7 +271,7 @@ func Run(options *RunOptions) error { // them to the appropriate slices. relPaths := map[string][]*setup.Slice{} for _, slice := range options.Selection.Slices { - arch := pkgArchive[slice.Package].Options().Arch + arch := pkgSources[slice.Package].Arch() for relPath, pathInfo := range slice.Contents { if len(pathInfo.Arch) > 0 && !slices.Contains(pathInfo.Arch, arch) { continue @@ -489,56 +490,3 @@ func createFile(targetDir, relPath string, pathInfo setup.PathInfo) (*fsutil.Ent MakeParents: true, }) } - -// selectPkgArchives selects the highest priority archive containing the package -// unless a particular archive is pinned within the slice definition file. It -// returns a map of archives indexed by package names. -func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Selection) (map[string]archive.Archive, error) { - sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) - for _, archive := range selection.Release.Archives { - if archive.Priority < 0 { - // Ignore negative priority archives unless a package specifically - // asks for it with the "archive" field. - continue - } - sortedArchives = append(sortedArchives, archive) - } - slices.SortFunc(sortedArchives, func(a, b *setup.Archive) int { - return b.Priority - a.Priority - }) - - pkgArchive := make(map[string]archive.Archive) - for _, s := range selection.Slices { - if _, ok := pkgArchive[s.Package]; ok { - continue - } - pkg := selection.Release.Packages[s.Package] - - if pkg.Store != "" { - return nil, fmt.Errorf("cannot fetch package %q from store %q: not implemented", pkg.Name, pkg.Store) - } - - var candidates []*setup.Archive - if pkg.Archive == "" { - // If the package has not pinned any archive, choose the highest - // priority archive in which the package exists. - candidates = sortedArchives - } else { - candidates = []*setup.Archive{selection.Release.Archives[pkg.Archive]} - } - - var chosen archive.Archive - for _, archiveInfo := range candidates { - archive := archives[archiveInfo.Name] - if archive != nil && archive.Exists(pkg.RealName) { - chosen = archive - break - } - } - if chosen == nil { - return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.RealName) - } - pkgArchive[pkg.Name] = chosen - } - return pkgArchive, nil -} diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d6ef9ca0d..954602de4 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1979,21 +1979,29 @@ var slicerTests = []slicerTest{{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, }, { - summary: "Store package is not yet implemented", - slices: []setup.SliceKey{{"bin-curl", "bin"}}, + summary: "Store package fetching not yet implemented", + slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, + arch: "amd64", release: map[string]string{ "chisel.yaml": testutil.DefaultChiselYamlWithStores, - "slices/curl.yaml": ` - package: curl + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/file: + `, + "slices/mydir/store-pkg.yaml": ` + package: store-pkg store: bin - default-track: latest + default-track: 3.1 slices: - bin: + myslice: contents: - /usr/bin/curl: + /dir/store-file: `, }, - error: `cannot fetch package "bin-curl" from store "bin": not implemented`, + error: `cannot fetch package "bin-store-pkg" from store "bin": not implemented`, }} func (s *S) TestRun(c *C) { diff --git a/internal/source/source.go b/internal/source/source.go new file mode 100644 index 000000000..217b5db6d --- /dev/null +++ b/internal/source/source.go @@ -0,0 +1,110 @@ +package source + +import ( + "fmt" + "io" + "slices" + + "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/setup" +) + +// Source is a resolved package source, abstracting over archives and stores. +type Source interface { + Arch() string + Fetch() (io.ReadSeekCloser, *archive.PackageInfo, error) +} + +// archiveSource adapts an archive.Archive to the Source interface for a +// specific package. +type archiveSource struct { + archive archive.Archive + name string +} + +func (a *archiveSource) Arch() string { + return a.archive.Options().Arch +} + +func (a *archiveSource) Fetch() (io.ReadSeekCloser, *archive.PackageInfo, error) { + return a.archive.Fetch(a.name) +} + +// storeSource adapts a store to the Source interface for a specific package. +type storeSource struct { + arch string + name string + store string +} + +func (s *storeSource) Arch() string { + return s.arch +} + +func (s *storeSource) Fetch() (io.ReadSeekCloser, *archive.PackageInfo, error) { + return nil, nil, fmt.Errorf("cannot fetch package %q from store %q: not implemented", s.name, s.store) +} + +// Resolve determines the source for each package in the selection. +// For archive packages it selects the highest priority archive containing the +// package unless a particular archive is pinned within the slice definition +// file. For store packages it records a store source. It returns a map of +// Source indexed by package names. +func Resolve(archives map[string]archive.Archive, selection *setup.Selection) (map[string]Source, error) { + sortedArchives := make([]*setup.Archive, 0, len(selection.Release.Archives)) + for _, archive := range selection.Release.Archives { + if archive.Priority < 0 { + // Ignore negative priority archives unless a package specifically + // asks for it with the "archive" field. + continue + } + sortedArchives = append(sortedArchives, archive) + } + slices.SortFunc(sortedArchives, func(a, b *setup.Archive) int { + return b.Priority - a.Priority + }) + + sources := make(map[string]Source) + for _, s := range selection.Slices { + if _, ok := sources[s.Package]; ok { + continue + } + pkg := selection.Release.Packages[s.Package] + if pkg.Store != "" { + sources[pkg.Name] = &storeSource{ + name: pkg.Name, + store: pkg.Store, + // TODO: populate arch, track and risk when implementing + // fetching from the store. + } + continue + } + + var candidates []*setup.Archive + if pkg.Archive == "" { + // If the package has not pinned any archive, choose the highest + // priority archive in which the package exists. + candidates = sortedArchives + } else { + candidates = []*setup.Archive{selection.Release.Archives[pkg.Archive]} + } + + var chosen archive.Archive + for _, archiveInfo := range candidates { + archive := archives[archiveInfo.Name] + if archive != nil && archive.Exists(pkg.RealName) { + chosen = archive + break + } + } + if chosen == nil { + return nil, fmt.Errorf("cannot find package %q in archive(s)", pkg.RealName) + } + sources[pkg.Name] = &archiveSource{ + archive: chosen, + name: pkg.RealName, + } + } + + return sources, nil +} diff --git a/internal/source/source_test.go b/internal/source/source_test.go new file mode 100644 index 000000000..edf3ed925 --- /dev/null +++ b/internal/source/source_test.go @@ -0,0 +1,275 @@ +package source_test + +import ( + "os" + "path/filepath" + "slices" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/source" + "github.com/canonical/chisel/internal/testutil" +) + +var testKey = testutil.PGPKeys["key1"] + +type sourceTest struct { + summary string + arch string + release map[string]string + pkgs []*testutil.TestPackage + slices []setup.SliceKey + archs map[string]string + fetchErrors map[string]string + error string +} + +var sourceTests = []sourceTest{{ + summary: "Highest priority archive is selected", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "amd64", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlTwoArchives, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + archs: map[string]string{"test-package": "amd64"}, +}, { + summary: "Pinned archive bypasses higher priority", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "amd64", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }, { + Name: "test-package", + Hash: "h2", + Version: "v2", + Arch: "amd64", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from bar"), + }), + Archives: []string{"bar"}, + }}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlTwoArchives, + "slices/mydir/test-package.yaml": ` + package: test-package + archive: bar + slices: + myslice: + contents: + /file: + `, + }, + archs: map[string]string{"test-package": "amd64"}, +}, { + summary: "Pinned archive not available fails", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "amd64", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlTwoArchives, + "slices/mydir/test-package.yaml": ` + package: test-package + archive: bar + slices: + myslice: + contents: + /file: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "No archives have the package", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlTwoArchives, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "Negative priority archives are ignored when not explicitly pinned", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.MustMakeDeb([]testutil.TarEntry{ + testutil.Reg(0644, "./file", "from foo"), + }), + Archives: []string{"foo"}, + }}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": ` + format: v1 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + foo: + version: 22.04 + components: [main, universe] + suites: [jammy] + priority: -20 + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + ` + `, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /file: + `, + }, + error: `cannot find package "test-package" in archive\(s\)`, +}, { + summary: "Store package fetching not yet implemented", + slices: []setup.SliceKey{{"test-package", "myslice"}, {"bin-store-pkg", "myslice"}}, + arch: "amd64", + release: map[string]string{ + "chisel.yaml": testutil.DefaultChiselYamlWithStores, + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/file: + `, + "slices/mydir/store-pkg.yaml": ` + package: store-pkg + store: bin + default-track: 3.1 + slices: + myslice: + contents: + /dir/store-file: + `, + }, + fetchErrors: map[string]string{ + "bin-store-pkg": `cannot fetch package "bin-store-pkg" from store "bin": not implemented`, + }, +}} + +func (s *S) TestResolve(c *C) { + for _, test := range sourceTests { + c.Logf("Summary: %s", test.summary) + + if _, ok := test.release["chisel.yaml"]; !ok { + test.release["chisel.yaml"] = testutil.DefaultChiselYaml + } + if test.pkgs == nil { + test.pkgs = []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }} + } + for _, pkg := range test.pkgs { + if pkg.Arch == "" { + pkg.Arch = "arch" + } + if pkg.Hash == "" { + pkg.Hash = "hash" + } + if pkg.Version == "" { + pkg.Version = "version" + } + } + + releaseDir := c.MkDir() + for path, data := range test.release { + fpath := filepath.Join(releaseDir, path) + err := os.MkdirAll(filepath.Dir(fpath), 0755) + c.Assert(err, IsNil) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + c.Assert(err, IsNil) + } + + release, err := setup.ReadRelease(releaseDir) + c.Assert(err, IsNil) + + selection, err := setup.Select(release, test.slices, test.arch) + c.Assert(err, IsNil) + + archives := map[string]archive.Archive{} + for name, setupArchive := range release.Archives { + pkgs := make(map[string]*testutil.TestPackage) + for _, pkg := range test.pkgs { + if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { + pkgs[pkg.Name] = pkg + } + } + archive := &testutil.TestArchive{ + Opts: archive.Options{ + Label: setupArchive.Name, + Version: setupArchive.Version, + Suites: setupArchive.Suites, + Components: setupArchive.Components, + Pro: setupArchive.Pro, + Arch: test.arch, + }, + Packages: pkgs, + } + archives[name] = archive + } + + sources, err := source.Resolve(archives, selection) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + + for pkgName, arch := range test.archs { + c.Assert(sources[pkgName].Arch(), Equals, arch) + } + + for pkgName, fetchErr := range test.fetchErrors { + _, _, err := sources[pkgName].Fetch() + c.Assert(err, ErrorMatches, fetchErr) + } + } +} diff --git a/internal/source/suite_test.go b/internal/source/suite_test.go new file mode 100644 index 000000000..f720627f9 --- /dev/null +++ b/internal/source/suite_test.go @@ -0,0 +1,13 @@ +package source_test + +import ( + "testing" + + . "gopkg.in/check.v1" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) diff --git a/internal/testutil/defaults.go b/internal/testutil/defaults.go index 046d46360..2b4b318c9 100644 --- a/internal/testutil/defaults.go +++ b/internal/testutil/defaults.go @@ -27,3 +27,26 @@ var DefaultChiselYamlWithStores = strings.ReplaceAll(DefaultChiselYaml, "format: version: 26.10 default-prefix: "bin-" ` + +var DefaultChiselYamlTwoArchives = ` + format: v1 + maintenance: + standard: 2025-01-01 + end-of-life: 2100-01-01 + archives: + foo: + version: 22.04 + components: [main, universe] + suites: [jammy] + priority: 20 + public-keys: [test-key] + bar: + version: 22.04 + components: [main] + suites: [jammy] + priority: 10 + public-keys: [test-key] + public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t")