diff options
author | renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> | 2023-10-19 08:24:49 +0000 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-10-19 08:24:49 +0000 |
commit | ad1ab71b8c81fcb42f76f50514b82c50520ad6b2 (patch) | |
tree | 365309a6b803c89640512e8635c29dc2714f9e37 | |
parent | 68636416d75223c20624dd2a4d074983972543fd (diff) |
fix(deps): update github.com/containers/storage digest to 79aa304
Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
-rw-r--r-- | go.mod | 4 | ||||
-rw-r--r-- | go.sum | 8 | ||||
-rw-r--r-- | vendor/github.com/containers/storage/drivers/driver.go | 3 | ||||
-rw-r--r-- | vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go | 6 | ||||
-rw-r--r-- | vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go | 50 | ||||
-rw-r--r-- | vendor/github.com/containers/storage/drivers/overlay/overlay.go | 37 | ||||
-rw-r--r-- | vendor/github.com/containers/storage/pkg/chunked/cache_linux.go | 5 | ||||
-rw-r--r-- | vendor/github.com/containers/storage/pkg/chunked/dump/dump.go | 230 | ||||
-rw-r--r-- | vendor/github.com/containers/storage/pkg/chunked/storage_linux.go | 22 | ||||
-rw-r--r-- | vendor/github.com/klauspost/compress/README.md | 9 | ||||
-rw-r--r-- | vendor/github.com/klauspost/compress/flate/inflate.go | 66 | ||||
-rw-r--r-- | vendor/github.com/klauspost/compress/flate/inflate_gen.go | 34 | ||||
-rw-r--r-- | vendor/modules.txt | 5 |
13 files changed, 399 insertions, 80 deletions
@@ -20,7 +20,7 @@ require ( github.com/containers/libhvee v0.4.1-0.20231012183749-e51be96b4854 github.com/containers/ocicrypt v1.1.8 github.com/containers/psgo v1.8.0 - github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb + github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff github.com/coreos/go-systemd/v22 v22.5.0 github.com/coreos/stream-metadata-go v0.4.3 github.com/crc-org/vfkit v0.1.2-0.20230829083117-09e62065eb6e @@ -139,7 +139,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.17.0 // indirect + github.com/klauspost/compress v1.17.1 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/fs v0.1.0 // indirect @@ -273,8 +273,8 @@ github.com/containers/ocicrypt v1.1.8/go.mod h1:jM362hyBtbwLMWzXQZTlkjKGAQf/BN/L github.com/containers/psgo v1.8.0 h1:2loGekmGAxM9ir5OsXWEfGwFxorMPYnc6gEDsGFQvhY= github.com/containers/psgo v1.8.0/go.mod h1:T8ZxnX3Ur4RvnhxFJ7t8xJ1F48RhiZB4rSrOaR/qGHc= github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s= -github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb h1:fYLIShJAebUb9l2643IoD9jrj1WEyhux/4/V+2D2CXM= -github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb/go.mod h1:RepNBJN07JUzkeLwmAeRwUstBqisvf8UoW/E6TR1Pak= +github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff h1:/z85oN77tAw5UUEzymFH73nHIgNtGeL9YednTut9E8M= +github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff/go.mod h1:UMgAdUgejmMKJQqeRE+0X/3GP9O6rXRm+E4uS7QrFgk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -680,8 +680,8 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index 1fb04dc3e..ab32d652e 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -189,6 +189,9 @@ type DriverWithDifferOutput struct { BigData map[string][]byte TarSplit []byte TOCDigest digest.Digest + // Artifacts is a collection of additional artifacts + // generated by the differ that the storage driver can use. + Artifacts map[string]interface{} } type DifferOutputFormat int diff --git a/vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go b/vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go index 5cdbcff6c..347e4d35c 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go +++ b/vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go @@ -11,7 +11,7 @@ func composeFsSupported() bool { return false } -func generateComposeFsBlob(toc []byte, composefsDir string) error { +func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, composefsDir string) error { return fmt.Errorf("composefs is not supported") } @@ -19,6 +19,6 @@ func mountComposefsBlob(dataDir, mountPoint string) error { return fmt.Errorf("composefs is not supported") } -func enableVerityRecursive(path string) error { - return fmt.Errorf("composefs is not supported") +func enableVerityRecursive(path string) (map[string]string, error) { + return nil, fmt.Errorf("composefs is not supported") } diff --git a/vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go b/vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go index 2f45c0f68..26dd36866 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go +++ b/vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go @@ -4,7 +4,6 @@ package overlay import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -16,6 +15,7 @@ import ( "syscall" "unsafe" + "github.com/containers/storage/pkg/chunked/dump" "github.com/containers/storage/pkg/loopback" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -29,7 +29,7 @@ var ( func getComposeFsHelper() (string, error) { composeFsHelperOnce.Do(func() { - composeFsHelperPath, composeFsHelperErr = exec.LookPath("composefs-from-json") + composeFsHelperPath, composeFsHelperErr = exec.LookPath("mkcomposefs") }) return composeFsHelperPath, composeFsHelperErr } @@ -53,7 +53,23 @@ func enableVerity(description string, fd int) error { return nil } -func enableVerityRecursive(path string) error { +type verityDigest struct { + Fsv unix.FsverityDigest + Buf [64]byte +} + +func measureVerity(description string, fd int) (string, error) { + var digest verityDigest + digest.Fsv.Size = 64 + _, _, e1 := syscall.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.FS_IOC_MEASURE_VERITY), uintptr(unsafe.Pointer(&digest))) + if e1 != 0 { + return "", fmt.Errorf("failed to measure verity for %q: %w", description, e1) + } + return fmt.Sprintf("%x", digest.Buf[:digest.Fsv.Size]), nil +} + +func enableVerityRecursive(root string) (map[string]string, error) { + digests := make(map[string]string) walkFn := func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -71,24 +87,42 @@ func enableVerityRecursive(path string) error { if err := enableVerity(path, int(f.Fd())); err != nil { return err } + + verity, err := measureVerity(path, int(f.Fd())) + if err != nil { + return err + } + + relPath, err := filepath.Rel(root, path) + if err != nil { + return err + } + + digests[relPath] = verity return nil } - return filepath.WalkDir(path, walkFn) + err := filepath.WalkDir(root, walkFn) + return digests, err } func getComposefsBlob(dataDir string) string { return filepath.Join(dataDir, "composefs.blob") } -func generateComposeFsBlob(toc []byte, composefsDir string) error { +func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, composefsDir string) error { if err := os.MkdirAll(composefsDir, 0o700); err != nil { return err } + dumpReader, err := dump.GenerateDump(toc, verityDigests) + if err != nil { + return err + } + destFile := getComposefsBlob(composefsDir) writerJson, err := getComposeFsHelper() if err != nil { - return fmt.Errorf("failed to find composefs-from-json: %w", err) + return fmt.Errorf("failed to find mkcomposefs: %w", err) } fd, err := unix.Openat(unix.AT_FDCWD, destFile, unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_EXCL|unix.O_CLOEXEC, 0o644) @@ -109,10 +143,10 @@ func generateComposeFsBlob(toc []byte, composefsDir string) error { // a scope to close outFd before setting fsverity on the read-only fd. defer outFd.Close() - cmd := exec.Command(writerJson, "--format=erofs", "--out=/proc/self/fd/3", "/proc/self/fd/0") + cmd := exec.Command(writerJson, "--from-file", "-", "/proc/self/fd/3") cmd.ExtraFiles = []*os.File{outFd} cmd.Stderr = os.Stderr - cmd.Stdin = bytes.NewReader(toc) + cmd.Stdin = dumpReader if err := cmd.Run(); err != nil { return fmt.Errorf("failed to convert json to erofs: %w", err) } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 525655331..04ecf871f 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -82,7 +82,7 @@ const ( lowerFile = "lower" maxDepth = 500 - zstdChunkedManifest = "zstd-chunked-manifest" + tocArtifact = "toc" // idLength represents the number of random characters // which can be used to create the unique link identifier @@ -1003,8 +1003,10 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, disable } } if parent != "" { - parentBase, parentImageStore, _ := d.dir2(parent) - if parentImageStore != "" { + parentBase, parentImageStore, inAdditionalStore := d.dir2(parent) + // If parentBase path is additional image store, select the image contained in parentBase. + // See https://github.com/containers/podman/issues/19748 + if parentImageStore != "" && !inAdditionalStore { parentBase = parentImageStore } st, err := system.Stat(filepath.Join(parentBase, "diff")) @@ -1079,12 +1081,13 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, disable } if parent != "" { - parentDir, parentImageStore, _ := d.dir2(parent) - base := parentDir - if parentImageStore != "" { - base = parentImageStore + parentBase, parentImageStore, inAdditionalStore := d.dir2(parent) + // If parentBase path is additional image store, select the image contained in parentBase. + // See https://github.com/containers/podman/issues/19748 + if parentImageStore != "" && !inAdditionalStore { + parentBase = parentImageStore } - st, err := system.Stat(filepath.Join(base, "diff")) + st, err := system.Stat(filepath.Join(parentBase, "diff")) if err != nil { return err } @@ -1526,15 +1529,8 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO defer cleanupFunc() } - composefsLayers := filepath.Join(workDirBase, "composefs-layers") - if err := os.MkdirAll(composefsLayers, 0o700); err != nil { - return "", err - } - skipIDMappingLayers := make(map[string]string) - composeFsLayers := []string{} - composefsMounts := []string{} defer func() { for _, m := range composefsMounts { @@ -1542,6 +1538,8 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } }() + composeFsLayers := []string{} + composeFsLayersDir := filepath.Join(workDirBase, "composefs-layers") maybeAddComposefsMount := func(lowerID string, i int, readWrite bool) (string, error) { composefsBlob := d.getComposefsData(lowerID) _, err = os.Stat(composefsBlob) @@ -1557,7 +1555,7 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO return "", fmt.Errorf("cannot mount a composefs layer as writeable") } - dest := filepath.Join(composefsLayers, fmt.Sprintf("%d", i)) + dest := filepath.Join(composeFsLayersDir, fmt.Sprintf("%d", i)) if err := os.MkdirAll(dest, 0o700); err != nil { return "", err } @@ -2110,11 +2108,12 @@ func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory stri if d.useComposeFs() { // FIXME: move this logic into the differ so we don't have to open // the file twice. - if err := enableVerityRecursive(stagingDirectory); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.ENOTTY) { + verityDigests, err := enableVerityRecursive(stagingDirectory) + if err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.ENOTTY) { logrus.Warningf("%s", err) } - toc := diffOutput.BigData[zstdChunkedManifest] - if err := generateComposeFsBlob(toc, d.getComposefsData(id)); err != nil { + toc := diffOutput.Artifacts[tocArtifact] + if err := generateComposeFsBlob(verityDigests, toc, d.getComposefsData(id)); err != nil { return err } } diff --git a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go index 56c30e267..5d4befc23 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go @@ -26,6 +26,8 @@ import ( const ( cacheKey = "chunked-manifest-cache" cacheVersion = 1 + + digestSha256Empty = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ) type metadata struct { @@ -650,6 +652,9 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) { iter.Skip() } } + if m.Type == TypeReg && m.Size == 0 && m.Digest == "" { + m.Digest = digestSha256Empty + } toc.Entries = append(toc.Entries, m) } break diff --git a/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go b/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go new file mode 100644 index 000000000..a08928034 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go @@ -0,0 +1,230 @@ +package dump + +import ( + "bufio" + "fmt" + "io" + "strings" + "time" + "unicode" + + "github.com/containers/storage/pkg/chunked/internal" + "golang.org/x/sys/unix" +) + +const ( + ESCAPE_STANDARD = 0 + NOESCAPE_SPACE = 1 << iota + ESCAPE_EQUAL + ESCAPE_LONE_DASH +) + +func escaped(val string, escape int) string { + noescapeSpace := escape&NOESCAPE_SPACE != 0 + escapeEqual := escape&ESCAPE_EQUAL != 0 + escapeLoneDash := escape&ESCAPE_LONE_DASH != 0 + + length := len(val) + + if escapeLoneDash && val == "-" { + return fmt.Sprintf("\\x%.2x", val[0]) + } + + var result string + for i := 0; i < length; i++ { + c := val[i] + hexEscape := false + var special string + + switch c { + case '\\': + special = "\\\\" + case '\n': + special = "\\n" + case '\r': + special = "\\r" + case '\t': + special = "\\t" + case '=': + hexEscape = escapeEqual + default: + if noescapeSpace { + hexEscape = !unicode.IsPrint(rune(c)) + } else { + hexEscape = !unicode.IsGraphic(rune(c)) + } + } + + if special != "" { + result += special + } else if hexEscape { + result += fmt.Sprintf("\\x%.2x", c) + } else { + result += string(c) + } + } + return result +} + +func escapedOptional(val string, escape int) string { + if val == "" { + return "-" + } + return escaped(val, escape) +} + +func getStMode(mode uint32, typ string) (uint32, error) { + switch typ { + case internal.TypeReg, internal.TypeLink: + mode |= unix.S_IFREG + case internal.TypeChar: + mode |= unix.S_IFCHR + case internal.TypeBlock: + mode |= unix.S_IFBLK + case internal.TypeDir: + mode |= unix.S_IFDIR + case internal.TypeFifo: + mode |= unix.S_IFIFO + case internal.TypeSymlink: + mode |= unix.S_IFLNK + default: + return 0, fmt.Errorf("unknown type %s", typ) + } + return mode, nil +} + +func dumpNode(out io.Writer, links map[string]int, verityDigests map[string]string, entry *internal.FileMetadata) error { + path := entry.Name + if path == "" { + path = "/" + } else if path[0] != '/' { + path = "/" + path + } + + if _, err := fmt.Fprint(out, escaped(path, ESCAPE_STANDARD)); err != nil { + return err + } + + nlinks := links[entry.Name] + links[entry.Linkname] + 1 + link := "" + if entry.Type == internal.TypeLink { + link = "@" + } + + rdev := unix.Mkdev(uint32(entry.Devmajor), uint32(entry.Devminor)) + + entryTime := entry.ModTime + if entryTime == nil { + t := time.Unix(0, 0) + entryTime = &t + } + + mode, err := getStMode(uint32(entry.Mode), entry.Type) + if err != nil { + return err + } + + if _, err := fmt.Fprintf(out, " %d %s%o %d %d %d %d %d.%d ", entry.Size, + link, mode, + nlinks, entry.UID, entry.GID, rdev, + entryTime.Unix(), entryTime.Nanosecond()); err != nil { + return err + } + + var payload string + if entry.Linkname != "" { + payload = entry.Linkname + if entry.Type == internal.TypeLink && payload[0] != '/' { + payload = "/" + payload + } + } else { + if len(entry.Digest) > 10 { + d := strings.Replace(entry.Digest, "sha256:", "", 1) + payload = d[:2] + "/" + d[2:] + } + } + + if _, err := fmt.Fprintf(out, escapedOptional(payload, ESCAPE_LONE_DASH)); err != nil { + return err + } + + /* inline content. */ + if _, err := fmt.Fprint(out, " -"); err != nil { + return err + } + + /* store the digest. */ + if _, err := fmt.Fprint(out, " "); err != nil { + return err + } + digest := verityDigests[payload] + if _, err := fmt.Fprintf(out, escapedOptional(digest, ESCAPE_LONE_DASH)); err != nil { + return err + } + + for k, v := range entry.Xattrs { + name := escaped(k, ESCAPE_EQUAL) + value := escaped(v, ESCAPE_EQUAL) + + if _, err := fmt.Fprintf(out, " %s=%s", name, value); err != nil { + return err + } + } + if _, err := fmt.Fprint(out, "\n"); err != nil { + return err + } + return nil +} + +// GenerateDump generates a dump of the TOC in the same format as `composefs-info dump` +func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, error) { + toc, ok := tocI.(*internal.TOC) + if !ok { + return nil, fmt.Errorf("invalid TOC type") + } + pipeR, pipeW := io.Pipe() + go func() { + closed := false + w := bufio.NewWriter(pipeW) + defer func() { + if !closed { + w.Flush() + pipeW.Close() + } + }() + + links := make(map[string]int) + for _, e := range toc.Entries { + if e.Linkname == "" { + continue + } + links[e.Linkname] = links[e.Linkname] + 1 + } + + if len(toc.Entries) == 0 || (toc.Entries[0].Name != "" && toc.Entries[0].Name != "/") { + root := &internal.FileMetadata{ + Name: "/", + Type: internal.TypeDir, + Mode: 0o755, + } + + if err := dumpNode(w, links, verityDigests, root); err != nil { + pipeW.CloseWithError(err) + closed = true + return + } + } + + for _, e := range toc.Entries { + if e.Type == internal.TypeChunk { + continue + } + if err := dumpNode(w, links, verityDigests, &e); err != nil { + pipeW.CloseWithError(err) + closed = true + return + } + } + }() + return pipeR, nil +} diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go index 088c92782..8493a2c19 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go @@ -45,6 +45,7 @@ const ( bigDataKey = "zstd-chunked-manifest" chunkedData = "zstd-chunked-data" chunkedLayerDataKey = "zstd-chunked-layer-data" + tocKey = "toc" fileTypeZstdChunked = iota fileTypeEstargz @@ -1470,10 +1471,7 @@ func makeEntriesFlat(mergedEntries []internal.FileMetadata) ([]internal.FileMeta continue } if mergedEntries[i].Digest == "" { - if mergedEntries[i].Size != 0 { - return nil, fmt.Errorf("missing digest for %q", mergedEntries[i].Name) - } - continue + return nil, fmt.Errorf("missing digest for %q", mergedEntries[i].Name) } digest, err := digest.Parse(mergedEntries[i].Digest) if err != nil { @@ -1542,6 +1540,13 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff if err != nil { return graphdriver.DriverWithDifferOutput{}, err } + + // Generate the manifest + toc, err := unmarshalToc(c.manifest) + if err != nil { + return graphdriver.DriverWithDifferOutput{}, err + } + output := graphdriver.DriverWithDifferOutput{ Differ: c, TarSplit: c.tarSplit, @@ -1549,6 +1554,9 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff bigDataKey: c.manifest, chunkedLayerDataKey: lcdBigData, }, + Artifacts: map[string]interface{}{ + tocKey: toc, + }, TOCDigest: c.contentDigest, } @@ -1563,12 +1571,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff // List of OSTree repositories to use for deduplication ostreeRepos := strings.Split(c.storeOpts.PullOptions["ostree_repos"], ":") - // Generate the manifest - toc, err := unmarshalToc(c.manifest) - if err != nil { - return output, err - } - whiteoutConverter := archive.GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData) var missingParts []missingPart diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index dde75389f..43de48677 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -16,6 +16,14 @@ This package provides various compression algorithms. # changelog
+* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0)
+ * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853
+ * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838
+ * flate: Add limited window compression https://github.com/klauspost/compress/pull/843
+ * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
+ * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
+ * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
+
* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7)
* zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829
* s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832
@@ -646,6 +654,7 @@ Here are other packages of good quality and pure Go (no cgo wrappers or autoconv * [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression.
* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression.
* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index.
+* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor.
# license
diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go index 414c0bea9..2f410d64f 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ b/vendor/github.com/klauspost/compress/flate/inflate.go @@ -120,8 +120,9 @@ func (h *huffmanDecoder) init(lengths []int) bool { const sanity = false if h.chunks == nil { - h.chunks = &[huffmanNumChunks]uint16{} + h.chunks = new([huffmanNumChunks]uint16) } + if h.maxRead != 0 { *h = huffmanDecoder{chunks: h.chunks, links: h.links} } @@ -175,6 +176,7 @@ func (h *huffmanDecoder) init(lengths []int) bool { } h.maxRead = min + chunks := h.chunks[:] for i := range chunks { chunks[i] = 0 @@ -202,8 +204,7 @@ func (h *huffmanDecoder) init(lengths []int) bool { if cap(h.links[off]) < numLinks { h.links[off] = make([]uint16, numLinks) } else { - links := h.links[off][:0] - h.links[off] = links[:numLinks] + h.links[off] = h.links[off][:numLinks] } } } else { @@ -277,7 +278,7 @@ func (h *huffmanDecoder) init(lengths []int) bool { return true } -// The actual read interface needed by NewReader. +// Reader is the actual read interface needed by NewReader. // If the passed in io.Reader does not also have ReadByte, // the NewReader will introduce its own buffering. type Reader interface { @@ -285,6 +286,18 @@ type Reader interface { io.ByteReader } +type step uint8 + +const ( + copyData step = iota + 1 + nextBlock + huffmanBytesBuffer + huffmanBytesReader + huffmanBufioReader + huffmanStringsReader + huffmanGenericReader +) + // Decompress state. type decompressor struct { // Input source. @@ -303,7 +316,7 @@ type decompressor struct { // Next step in the decompression, // and decompression state. - step func(*decompressor) + step step stepState int err error toRead []byte @@ -342,7 +355,7 @@ func (f *decompressor) nextBlock() { // compressed, fixed Huffman tables f.hl = &fixedHuffmanDecoder f.hd = nil - f.huffmanBlockDecoder()() + f.huffmanBlockDecoder() if debugDecode { fmt.Println("predefinied huffman block") } @@ -353,7 +366,7 @@ func (f *decompressor) nextBlock() { } f.hl = &f.h1 f.hd = &f.h2 - f.huffmanBlockDecoder()() + f.huffmanBlockDecoder() if debugDecode { fmt.Println("dynamic huffman block") } @@ -379,14 +392,16 @@ func (f *decompressor) Read(b []byte) (int, error) { if f.err != nil { return 0, f.err } - f.step(f) + + f.doStep() + if f.err != nil && len(f.toRead) == 0 { f.toRead = f.dict.readFlush() // Flush what's left in case of error } } } -// Support the io.WriteTo interface for io.Copy and friends. +// WriteTo implements the io.WriteTo interface for io.Copy and friends. func (f *decompressor) WriteTo(w io.Writer) (int64, error) { total := int64(0) flushed := false @@ -410,7 +425,7 @@ func (f *decompressor) WriteTo(w io.Writer) (int64, error) { return total, f.err } if f.err == nil { - f.step(f) + f.doStep() } if len(f.toRead) == 0 && f.err != nil && !flushed { f.toRead = f.dict.readFlush() // Flush what's left in case of error @@ -631,7 +646,7 @@ func (f *decompressor) copyData() { if f.dict.availWrite() == 0 || f.copyLen > 0 { f.toRead = f.dict.readFlush() - f.step = (*decompressor).copyData + f.step = copyData return } f.finishBlock() @@ -644,7 +659,28 @@ func (f *decompressor) finishBlock() { } f.err = io.EOF } - f.step = (*decompressor).nextBlock + f.step = nextBlock +} + +func (f *decompressor) doStep() { + switch f.step { + case copyData: + f.copyData() + case nextBlock: + f.nextBlock() + case huffmanBytesBuffer: + f.huffmanBytesBuffer() + case huffmanBytesReader: + f.huffmanBytesReader() + case huffmanBufioReader: + f.huffmanBufioReader() + case huffmanStringsReader: + f.huffmanStringsReader() + case huffmanGenericReader: + f.huffmanGenericReader() + default: + panic("BUG: unexpected step state") + } } // noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF. @@ -747,7 +783,7 @@ func (f *decompressor) Reset(r io.Reader, dict []byte) error { h1: f.h1, h2: f.h2, dict: f.dict, - step: (*decompressor).nextBlock, + step: nextBlock, } f.dict.init(maxMatchOffset, dict) return nil @@ -768,7 +804,7 @@ func NewReader(r io.Reader) io.ReadCloser { f.r = makeReader(r) f.bits = new([maxNumLit + maxNumDist]int) f.codebits = new([numCodes]int) - f.step = (*decompressor).nextBlock + f.step = nextBlock f.dict.init(maxMatchOffset, nil) return &f } @@ -787,7 +823,7 @@ func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { f.r = makeReader(r) f.bits = new([maxNumLit + maxNumDist]int) f.codebits = new([numCodes]int) - f.step = (*decompressor).nextBlock + f.step = nextBlock f.dict.init(maxMatchOffset, dict) return &f } diff --git a/vendor/github.com/klauspost/compress/flate/inflate_gen.go b/vendor/github.com/klauspost/compress/flate/inflate_gen.go index 61342b6b8..2b2f993f7 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate_gen.go +++ b/vendor/github.com/klauspost/compress/flate/inflate_gen.go @@ -85,7 +85,7 @@ readLiteral: dict.writeByte(byte(v)) if dict.availWrite() == 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanBytesBuffer + f.step = huffmanBytesBuffer f.stepState = stateInit f.b, f.nb = fb, fnb return @@ -251,7 +251,7 @@ copyHistory: if dict.availWrite() == 0 || f.copyLen > 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanBytesBuffer // We need to continue this work + f.step = huffmanBytesBuffer // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb return @@ -336,7 +336,7 @@ readLiteral: dict.writeByte(byte(v)) if dict.availWrite() == 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanBytesReader + f.step = huffmanBytesReader f.stepState = stateInit f.b, f.nb = fb, fnb return @@ -502,7 +502,7 @@ copyHistory: if dict.availWrite() == 0 || f.copyLen > 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanBytesReader // We need to continue this work + f.step = huffmanBytesReader // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb return @@ -587,7 +587,7 @@ readLiteral: dict.writeByte(byte(v)) if dict.availWrite() == 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanBufioReader + f.step = huffmanBufioReader f.stepState = stateInit f.b, f.nb = fb, fnb return @@ -753,7 +753,7 @@ copyHistory: if dict.availWrite() == 0 || f.copyLen > 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanBufioReader // We need to continue this work + f.step = huffmanBufioReader // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb return @@ -838,7 +838,7 @@ readLiteral: dict.writeByte(byte(v)) if dict.availWrite() == 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanStringsReader + f.step = huffmanStringsReader f.stepState = stateInit f.b, f.nb = fb, fnb return @@ -1004,7 +1004,7 @@ copyHistory: if dict.availWrite() == 0 || f.copyLen > 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanStringsReader // We need to continue this work + f.step = huffmanStringsReader // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb return @@ -1089,7 +1089,7 @@ readLiteral: dict.writeByte(byte(v)) if dict.availWrite() == 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanGenericReader + f.step = huffmanGenericReader f.stepState = stateInit f.b, f.nb = fb, fnb return @@ -1255,7 +1255,7 @@ copyHistory: if dict.availWrite() == 0 || f.copyLen > 0 { f.toRead = dict.readFlush() - f.step = (*decompressor).huffmanGenericReader // We need to continue this work + f.step = huffmanGenericReader // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb return @@ -1265,19 +1265,19 @@ copyHistory: // Not reached } -func (f *decompressor) huffmanBlockDecoder() func() { +func (f *decompressor) huffmanBlockDecoder() { switch f.r.(type) { case *bytes.Buffer: - return f.huffmanBytesBuffer + f.huffmanBytesBuffer() case *bytes.Reader: - return f.huffmanBytesReader + f.huffmanBytesReader() case *bufio.Reader: - return f.huffmanBufioReader + f.huffmanBufioReader() case *strings.Reader: - return f.huffmanStringsReader + f.huffmanStringsReader() case Reader: - return f.huffmanGenericReader + f.huffmanGenericReader() default: - return f.huffmanGenericReader + f.huffmanGenericReader() } } diff --git a/vendor/modules.txt b/vendor/modules.txt index 0861649ec..4c03bf8c3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -343,7 +343,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb +# github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff ## explicit; go 1.19 github.com/containers/storage github.com/containers/storage/drivers @@ -362,6 +362,7 @@ github.com/containers/storage/pkg/archive github.com/containers/storage/pkg/chrootarchive github.com/containers/storage/pkg/chunked github.com/containers/storage/pkg/chunked/compressor +github.com/containers/storage/pkg/chunked/dump github.com/containers/storage/pkg/chunked/internal github.com/containers/storage/pkg/config github.com/containers/storage/pkg/devicemapper @@ -688,7 +689,7 @@ github.com/josharian/intern # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/klauspost/compress v1.17.0 +# github.com/klauspost/compress v1.17.1 ## explicit; go 1.18 github.com/klauspost/compress github.com/klauspost/compress/flate |