From a3ba19aeaa9fbcf8aba1e45ae4abade88957f04c Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 29 Sep 2026 17:34:41 +0200 Subject: [PATCH] Split AddData chunks into 4MB chunks The readers we have (go, rust) are limited to 100MB, and I got this when applying a delta: ``` Reconstructing layer sha256:4017c801fd2d816d6347cbee0644b10009fbae0d01f160fafe5221261f0b8593: AddData size 126080000 exceeds limit ``` So, lets split chunks up to avoid this. Signed-off-by: Alexander Larsson --- pkg/tar-diff/delta.go | 15 ++++++++++----- pkg/tar-diff/delta_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pkg/tar-diff/delta.go b/pkg/tar-diff/delta.go index 89de1a3..d1b295b 100644 --- a/pkg/tar-diff/delta.go +++ b/pkg/tar-diff/delta.go @@ -191,12 +191,17 @@ func (d *deltaWriter) WriteAddContent(data []byte) error { return err } - size := uint64(len(data)) - err = d.writeOp(protocol.DeltaOpAddData, size, data) - if err != nil { - return err + for len(data) > 0 { + chunk := data + if len(chunk) > deltaDataChunkSize { + chunk = data[:deltaDataChunkSize] + } + if err := d.writeOp(protocol.DeltaOpAddData, uint64(len(chunk)), chunk); err != nil { + return err + } + d.currentPos += uint64(len(chunk)) + data = data[len(chunk):] } - d.currentPos += size return nil } diff --git a/pkg/tar-diff/delta_test.go b/pkg/tar-diff/delta_test.go index 9574781..359f0ac 100644 --- a/pkg/tar-diff/delta_test.go +++ b/pkg/tar-diff/delta_test.go @@ -265,6 +265,40 @@ func TestDeltaWriterWriteAddContent(t *testing.T) { } } +func TestDeltaWriterWriteAddContentChunks(t *testing.T) { + var output bytes.Buffer + deltaWriter, err := newDeltaWriter(&output, 1, deltaFormatV1) + if err != nil { + t.Fatal(err) + } + + data := make([]byte, 2*deltaDataChunkSize+123) + for i := range data { + data[i] = byte(i) + } + if err := deltaWriter.WriteAddContent(data); err != nil { + t.Fatal(err) + } + if deltaWriter.currentPos != uint64(len(data)) { + t.Fatalf("currentPos = %d, want %d", deltaWriter.currentPos, len(data)) + } + if err := deltaWriter.Close(); err != nil { + t.Fatal(err) + } + + ops, sizes := decodeDeltaOps(t, output.Bytes()) + wantSizes := []uint64{deltaDataChunkSize, deltaDataChunkSize, 123} + if len(ops) != len(wantSizes) { + t.Fatalf("got %d operations, want %d", len(ops), len(wantSizes)) + } + for i, op := range ops { + if op != protocol.DeltaOpAddData || sizes[i] != wantSizes[i] { + t.Errorf("operation %d = (op %d, size %d), want (op %d, size %d)", + i, op, sizes[i], protocol.DeltaOpAddData, wantSizes[i]) + } + } +} + func TestDeltaWriterWriteOldFile(t *testing.T) { var output bytes.Buffer