Allow `ReadOption`s to be set when getting or generating index

Fix a bug in read options where the value of zero-len option is always
set to `true` even if the flag passed is false. This meant if the option
was present it would have always enabled the zero-len as EOF.

Allow the user to specify read options on `ReadOrGenerateIndex`. Pass
the options onto downstream reader/generators.

Generate a CARv1 file with null padding and check it into `testdata`. In
addition check in another such file from ignite just to have two
different samples of this case.

Enhance index generation tests to expect error when zero-len option is
not set, and generate index when it is for files with null padding.

Make RW blockstore test use the fixed test files from `testdata` instead
of generating every time.

v2/blockstore/readwrite_test.go

@@ -305,12 +305,9 @@ func (b bufferReaderAt) ReadAt(p []byte, off int64) (int, error) {
 }
 
 func TestBlockstoreNullPadding(t *testing.T) {
-	paddedV1, err := ioutil.ReadFile("../testdata/sample-v1.car")
+	paddedV1, err := ioutil.ReadFile("../testdata/sample-v1-with-zero-len-section.car")
 	require.NoError(t, err)
 
-	// A sample null-padded CARv1 file.
-	paddedV1 = append(paddedV1, make([]byte, 2048)...)
-
 	rbs, err := blockstore.NewReadOnly(bufferReaderAt(paddedV1), nil,
 		carv2.ZeroLengthSectionAsEOF(true))
 	require.NoError(t, err)

v2/index_gen.go

@@ -110,7 +110,7 @@ func GenerateIndexFromFile(path string) (index.Index, error) {
 //
 // Note, the returned index lives entirely in memory and will not depend on the
 // given reader to fulfill index lookup.
-func ReadOrGenerateIndex(rs io.ReadSeeker) (index.Index, error) {
+func ReadOrGenerateIndex(rs io.ReadSeeker, opts ...ReadOption) (index.Index, error) {
 	// Read version.
 	version, err := ReadVersion(rs)
 	if err != nil {
@@ -124,10 +124,10 @@ func ReadOrGenerateIndex(rs io.ReadSeeker) (index.Index, error) {
 	switch version {
 	case 1:
 		// Simply generate the index, since there can't be a pre-existing one.
-		return GenerateIndex(rs)
+		return GenerateIndex(rs, opts...)
 	case 2:
 		// Read CARv2 format
-		v2r, err := NewReader(internalio.ToReaderAt(rs))
+		v2r, err := NewReader(internalio.ToReaderAt(rs), opts...)
 		if err != nil {
 			return nil, err
 		}
@@ -136,7 +136,7 @@ func ReadOrGenerateIndex(rs io.ReadSeeker) (index.Index, error) {
 			return index.ReadFrom(v2r.IndexReader())
 		}
 		// Otherwise, generate index from CARv1 payload wrapped within CARv2 format.
-		return GenerateIndex(v2r.DataReader())
+		return GenerateIndex(v2r.DataReader(), opts...)
 	default:
 		return nil, fmt.Errorf("unknown version %v", version)
 	}

v2/index_gen_test.go

@@ -13,12 +13,14 @@ func TestReadOrGenerateIndex(t *testing.T) {
 	tests := []struct {
 		name        string
 		carPath     string
+		readOpts    []ReadOption
 		wantIndexer func(t *testing.T) index.Index
 		wantErr     bool
 	}{
 		{
 			"CarV1IsIndexedAsExpected",
 			"testdata/sample-v1.car",
+			[]ReadOption{},
 			func(t *testing.T) index.Index {
 				v1, err := os.Open("testdata/sample-v1.car")
 				require.NoError(t, err)
@@ -32,6 +34,7 @@ func TestReadOrGenerateIndex(t *testing.T) {
 		{
 			"CarV2WithIndexIsReturnedAsExpected",
 			"testdata/sample-wrapped-v2.car",
+			[]ReadOption{},
 			func(t *testing.T) index.Index {
 				v2, err := os.Open("testdata/sample-wrapped-v2.car")
 				require.NoError(t, err)
@@ -44,9 +47,45 @@ func TestReadOrGenerateIndex(t *testing.T) {
 			},
 			false,
 		},
+		{
+			"CarV1WithZeroLenSectionIsGeneratedAsExpected",
+			"testdata/sample-v1-with-zero-len-section.car",
+			[]ReadOption{ZeroLengthSectionAsEOF(true)},
+			func(t *testing.T) index.Index {
+				v1, err := os.Open("testdata/sample-v1-with-zero-len-section.car")
+				require.NoError(t, err)
+				defer v1.Close()
+				want, err := GenerateIndex(v1, ZeroLengthSectionAsEOF(true))
+				require.NoError(t, err)
+				return want
+			},
+			false,
+		},
+		{
+			"AnotherCarV1WithZeroLenSectionIsGeneratedAsExpected",
+			"testdata/sample-v1-with-zero-len-section2.car",
+			[]ReadOption{ZeroLengthSectionAsEOF(true)},
+			func(t *testing.T) index.Index {
+				v1, err := os.Open("testdata/sample-v1-with-zero-len-section2.car")
+				require.NoError(t, err)
+				defer v1.Close()
+				want, err := GenerateIndex(v1, ZeroLengthSectionAsEOF(true))
+				require.NoError(t, err)
+				return want
+			},
+			false,
+		},
+		{
+			"CarV1WithZeroLenSectionWithoutOptionIsError",
+			"testdata/sample-v1-with-zero-len-section.car",
+			[]ReadOption{},
+			func(t *testing.T) index.Index { return nil },
+			true,
+		},
 		{
 			"CarOtherThanV1OrV2IsError",
 			"testdata/sample-rootless-v42.car",
+			[]ReadOption{},
 			func(t *testing.T) index.Index { return nil },
 			true,
 		},
@@ -56,7 +95,7 @@ func TestReadOrGenerateIndex(t *testing.T) {
 			carFile, err := os.Open(tt.carPath)
 			require.NoError(t, err)
 			t.Cleanup(func() { assert.NoError(t, carFile.Close()) })
-			got, err := ReadOrGenerateIndex(carFile)
+			got, err := ReadOrGenerateIndex(carFile, tt.readOpts...)
 			if tt.wantErr {
 				require.Error(t, err)
 			}

v2/options.go

@@ -48,7 +48,7 @@ type ReadWriteOption interface {
 // padding begins.
 func ZeroLengthSectionAsEOF(enable bool) ReadOption {
 	return func(o *ReadOptions) {
-		o.ZeroLengthSectionAsEOF = true
+		o.ZeroLengthSectionAsEOF = enable
 	}
 }