diff --git a/parse.go b/parse.go index 59656bf84695f6221dc93b2b597db6092d04c438..dee830489aa71ec58d32181f0081ad3057553f71 100644 --- a/parse.go +++ b/parse.go @@ -12,21 +12,16 @@ const ( // DefaultBlockSize is the chunk size that splitters produce (or aim to). DefaultBlockSize int64 = 1024 * 256 - // 1 MB, on-wire block size for "datablocks ( unixfs, etc )" - // copy of https://github.com/ipfs/go-unixfs/blob/v0.2.3/importer/helpers/helpers.go#L8 - BlockSizeLimit int = 1048576 - - // in case we are using raw-leaves: this would match BlockSizeLimit, but we can't assume that - // be conservative and substract the PB wraping size of a full DAG-PB+UnixFS node describing 1M - // (2b(type2/file)+4b(data-field:3-byte-len-delimited)+4b(size-field:3-byte-varint))+(4b(DAG-type-1:3-byte-len-delimited)) - // FIXME - this calculation will need an update for CBOR - BlockPayloadLimit int = (BlockSizeLimit - (2 + 4 + 4 + 4)) + // No leaf block should contain more than 1MiB of payload data ( wrapping overhead aside ) + // This effectively mandates the maximum chunk size + // See discussion at https://github.com/ipfs/go-ipfs-chunker/pull/21#discussion_r369124879 for background + ChunkSizeLimit int = 1048576 ) var ( ErrRabinMin = errors.New("rabin min must be greater than 16") ErrSize = errors.New("chunker size must be greater than 0") - ErrSizeMax = fmt.Errorf("chunker parameters may not exceed the maximum block payload size of %d", BlockPayloadLimit) + ErrSizeMax = fmt.Errorf("chunker parameters may not exceed the maximum chunk size of %d", ChunkSizeLimit) ) // FromString returns a Splitter depending on the given string: @@ -44,7 +39,7 @@ func FromString(r io.Reader, chunker string) (Splitter, error) { return nil, err } else if size <= 0 { return nil, ErrSize - } else if size > BlockPayloadLimit { + } else if size > ChunkSizeLimit { return nil, ErrSizeMax } return NewSizeSplitter(r, int64(size)), nil @@ -69,7 +64,7 @@ func parseRabinString(r io.Reader, chunker string) (Splitter, error) { size, err := strconv.Atoi(parts[1]) if err != nil { return nil, err - } else if int(float32(size)*1.5) > BlockPayloadLimit { // FIXME - there is probably a better way to bubble up this calculation from NewRabin() + } else if int(float32(size)*1.5) > ChunkSizeLimit { // FIXME - this will be addressed in a subsequent PR return nil, ErrSizeMax } return NewRabin(r, uint64(size)), nil @@ -108,7 +103,7 @@ func parseRabinString(r io.Reader, chunker string) (Splitter, error) { return nil, errors.New("incorrect format: rabin-min must be smaller than rabin-avg") } else if avg >= max { return nil, errors.New("incorrect format: rabin-avg must be smaller than rabin-max") - } else if max > BlockPayloadLimit { + } else if max > ChunkSizeLimit { return nil, ErrSizeMax } diff --git a/parse_test.go b/parse_test.go index 242d582eead1bae14f72a340a55ab96ab5dab935..237a2b439a6cc74a0a83a3b63da3affa0e9cee87 100644 --- a/parse_test.go +++ b/parse_test.go @@ -7,7 +7,7 @@ import ( ) const ( - testTwoThirdsOfBlockPayloadLimit = 2 * (float32(BlockPayloadLimit) / float32(3)) + testTwoThirdsOfChunkLimit = 2 * (float32(ChunkSizeLimit) / float32(3)) ) func TestParseRabin(t *testing.T) { @@ -33,22 +33,22 @@ func TestParseRabin(t *testing.T) { t.Fatalf("Expected an arg-out-of-order error, got: %#v", err) } - _, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", BlockPayloadLimit)) + _, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", ChunkSizeLimit)) if err != nil { t.Fatalf("Expected success, got: %#v", err) } - _, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", 1+BlockPayloadLimit)) + _, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", 1+ChunkSizeLimit)) if err != ErrSizeMax { t.Fatalf("Expected 'ErrSizeMax', got: %#v", err) } - _, err = FromString(r, fmt.Sprintf("rabin-%.0f", testTwoThirdsOfBlockPayloadLimit)) + _, err = FromString(r, fmt.Sprintf("rabin-%.0f", testTwoThirdsOfChunkLimit)) if err != nil { t.Fatalf("Expected success, got: %#v", err) } - _, err = FromString(r, fmt.Sprintf("rabin-%.0f", 1+testTwoThirdsOfBlockPayloadLimit)) + _, err = FromString(r, fmt.Sprintf("rabin-%.0f", 1+testTwoThirdsOfChunkLimit)) if err != ErrSizeMax { t.Fatalf("Expected 'ErrSizeMax', got: %#v", err) } @@ -68,12 +68,12 @@ func TestParseSize(t *testing.T) { t.Fatalf("Expected success, got: %#v", err) } - _, err = FromString(r, fmt.Sprintf("size-%d", BlockPayloadLimit)) + _, err = FromString(r, fmt.Sprintf("size-%d", ChunkSizeLimit)) if err != nil { t.Fatalf("Expected success, got: %#v", err) } - _, err = FromString(r, fmt.Sprintf("size-%d", 1+BlockPayloadLimit)) + _, err = FromString(r, fmt.Sprintf("size-%d", 1+ChunkSizeLimit)) if err != ErrSizeMax { t.Fatalf("Expected 'ErrSizeMax', got: %#v", err) }