Address block/chunk logical mismatch, name things correctly

parse.go

@@ -12,21 +12,16 @@ const (
 	// DefaultBlockSize is the chunk size that splitters produce (or aim to).
 	DefaultBlockSize int64 = 1024 * 256
 
-	// 1 MB, on-wire block size for "datablocks ( unixfs, etc )"
-	// copy of https://github.com/ipfs/go-unixfs/blob/v0.2.3/importer/helpers/helpers.go#L8
-	BlockSizeLimit int = 1048576
-
-	// in case we are using raw-leaves: this would match BlockSizeLimit, but we can't assume that
-	// be conservative and substract the PB wraping size of a full DAG-PB+UnixFS node describing 1M
-	// (2b(type2/file)+4b(data-field:3-byte-len-delimited)+4b(size-field:3-byte-varint))+(4b(DAG-type-1:3-byte-len-delimited))
-	// FIXME - this calculation will need an update for CBOR
-	BlockPayloadLimit int = (BlockSizeLimit - (2 + 4 + 4 + 4))
+	// No leaf block should contain more than 1MiB of payload data ( wrapping overhead aside )
+	// This effectively mandates the maximum chunk size
+	// See discussion at https://github.com/ipfs/go-ipfs-chunker/pull/21#discussion_r369124879 for background
+	ChunkSizeLimit int = 1048576
 )
 
 var (
 	ErrRabinMin = errors.New("rabin min must be greater than 16")
 	ErrSize     = errors.New("chunker size must be greater than 0")
-	ErrSizeMax  = fmt.Errorf("chunker parameters may not exceed the maximum block payload size of %d", BlockPayloadLimit)
+	ErrSizeMax  = fmt.Errorf("chunker parameters may not exceed the maximum chunk size of %d", ChunkSizeLimit)
 )
 
 // FromString returns a Splitter depending on the given string:
@@ -44,7 +39,7 @@ func FromString(r io.Reader, chunker string) (Splitter, error) {
 			return nil, err
 		} else if size <= 0 {
 			return nil, ErrSize
-		} else if size > BlockPayloadLimit {
+		} else if size > ChunkSizeLimit {
 			return nil, ErrSizeMax
 		}
 		return NewSizeSplitter(r, int64(size)), nil
@@ -69,7 +64,7 @@ func parseRabinString(r io.Reader, chunker string) (Splitter, error) {
 		size, err := strconv.Atoi(parts[1])
 		if err != nil {
 			return nil, err
-		} else if int(float32(size)*1.5) > BlockPayloadLimit { // FIXME - there is probably a better way to bubble up this calculation from NewRabin()
+		} else if int(float32(size)*1.5) > ChunkSizeLimit { // FIXME - this will be addressed in a subsequent PR
 			return nil, ErrSizeMax
 		}
 		return NewRabin(r, uint64(size)), nil
@@ -108,7 +103,7 @@ func parseRabinString(r io.Reader, chunker string) (Splitter, error) {
 			return nil, errors.New("incorrect format: rabin-min must be smaller than rabin-avg")
 		} else if avg >= max {
 			return nil, errors.New("incorrect format: rabin-avg must be smaller than rabin-max")
-		} else if max > BlockPayloadLimit {
+		} else if max > ChunkSizeLimit {
 			return nil, ErrSizeMax
 		}
 

parse_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 const (
-	testTwoThirdsOfBlockPayloadLimit = 2 * (float32(BlockPayloadLimit) / float32(3))
+	testTwoThirdsOfChunkLimit = 2 * (float32(ChunkSizeLimit) / float32(3))
 )
 
 func TestParseRabin(t *testing.T) {
@@ -33,22 +33,22 @@ func TestParseRabin(t *testing.T) {
 		t.Fatalf("Expected an arg-out-of-order error, got: %#v", err)
 	}
 
-	_, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", BlockPayloadLimit))
+	_, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", ChunkSizeLimit))
 	if err != nil {
 		t.Fatalf("Expected success, got: %#v", err)
 	}
 
-	_, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", 1+BlockPayloadLimit))
+	_, err = FromString(r, fmt.Sprintf("rabin-19-21-%d", 1+ChunkSizeLimit))
 	if err != ErrSizeMax {
 		t.Fatalf("Expected 'ErrSizeMax', got: %#v", err)
 	}
 
-	_, err = FromString(r, fmt.Sprintf("rabin-%.0f", testTwoThirdsOfBlockPayloadLimit))
+	_, err = FromString(r, fmt.Sprintf("rabin-%.0f", testTwoThirdsOfChunkLimit))
 	if err != nil {
 		t.Fatalf("Expected success, got: %#v", err)
 	}
 
-	_, err = FromString(r, fmt.Sprintf("rabin-%.0f", 1+testTwoThirdsOfBlockPayloadLimit))
+	_, err = FromString(r, fmt.Sprintf("rabin-%.0f", 1+testTwoThirdsOfChunkLimit))
 	if err != ErrSizeMax {
 		t.Fatalf("Expected 'ErrSizeMax', got: %#v", err)
 	}
@@ -68,12 +68,12 @@ func TestParseSize(t *testing.T) {
 		t.Fatalf("Expected success, got: %#v", err)
 	}
 
-	_, err = FromString(r, fmt.Sprintf("size-%d", BlockPayloadLimit))
+	_, err = FromString(r, fmt.Sprintf("size-%d", ChunkSizeLimit))
 	if err != nil {
 		t.Fatalf("Expected success, got: %#v", err)
 	}
 
-	_, err = FromString(r, fmt.Sprintf("size-%d", 1+BlockPayloadLimit))
+	_, err = FromString(r, fmt.Sprintf("size-%d", 1+ChunkSizeLimit))
 	if err != ErrSizeMax {
 		t.Fatalf("Expected 'ErrSizeMax', got: %#v", err)
 	}