diff --git a/v2/blockstore/readonly_test.go b/v2/blockstore/readonly_test.go index 97ed730912eab026dfae00fdfc72aeb529b345a9..10f9804cd9a6a7d4f981e14b30a5786cbc245af9 100644 --- a/v2/blockstore/readonly_test.go +++ b/v2/blockstore/readonly_test.go @@ -2,13 +2,14 @@ package blockstore import ( "context" - blockstore "github.com/ipfs/go-ipfs-blockstore" - "github.com/ipfs/go-merkledag" "io" "os" "testing" "time" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-merkledag" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" diff --git a/v2/index/example_test.go b/v2/index/example_test.go new file mode 100644 index 0000000000000000000000000000000000000000..cf6d56a630b50168f21f559294740317e570ca51 --- /dev/null +++ b/v2/index/example_test.go @@ -0,0 +1,93 @@ +package index_test + +import ( + "fmt" + "os" + "reflect" + + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/index" +) + +// ExampleReadFrom unmarshalls an index from an indexed CARv2 file, and for each root CID prints the +// offset at which its corresponding block starts relative to the wrapped CARv1 data payload. +func ExampleReadFrom() { + // Open the CARv2 file + cr, err := carv2.OpenReader("../testdata/sample-wrapped-v2.car") + if err != nil { + panic(err) + } + defer cr.Close() + + // Get root CIDs in the CARv1 file. + roots, err := cr.Roots() + if err != nil { + panic(err) + } + + // Read and unmarshall index within CARv2 file. + idx, err := index.ReadFrom(cr.IndexReader()) + if err != nil { + panic(err) + } + + // For each root CID print the offset relative to CARv1 data payload. + for _, r := range roots { + offset, err := idx.Get(r) + if err != nil { + panic(err) + } + fmt.Printf("Frame with CID %v starts at offset %v relative to CARv1 data payload.\n", r, offset) + } + + // Output: + // Frame with CID bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy starts at offset 61 relative to CARv1 data payload. +} + +// ExampleSave unmarshalls an index from an indexed CARv2 file, and stores it as a separate +// file on disk. +func ExampleSave() { + // Open the CARv2 file + src := "../testdata/sample-wrapped-v2.car" + cr, err := carv2.OpenReader(src) + if err != nil { + panic(err) + } + defer cr.Close() + + // Read and unmarshall index within CARv2 file. + idx, err := index.ReadFrom(cr.IndexReader()) + if err != nil { + panic(err) + } + + // Store the index alone onto destination file. + dest := "../testdata/sample-index.carindex" + err = index.Save(idx, dest) + if err != nil { + panic(err) + } + + // Open the destination file that contains the index only. + f, err := os.Open(dest) + if err != nil { + panic(err) + } + defer f.Close() + + // Read and unmarshall the destination file as a separate index instance. + reReadIdx, err := index.ReadFrom(f) + if err != nil { + panic(err) + } + + // Expect indices to be equal. + if reflect.DeepEqual(idx, reReadIdx) { + fmt.Printf("Saved index file at %v matches the index embedded in CARv2 at %v.\n", dest, src) + } else { + panic("expected to get the same index as the CARv2 file") + } + + // Output: + // Saved index file at ../testdata/sample-index.carindex matches the index embedded in CARv2 at ../testdata/sample-wrapped-v2.car. +} diff --git a/v2/index/index_test.go b/v2/index/index_test.go new file mode 100644 index 0000000000000000000000000000000000000000..945ca1bbc17769768cdc013061944715a6947fc6 --- /dev/null +++ b/v2/index/index_test.go @@ -0,0 +1,172 @@ +package index + +import ( + "bytes" + "io" + "os" + "path/filepath" + "testing" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipld/go-car/v2/internal/carv1" + "github.com/ipld/go-car/v2/internal/carv1/util" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-varint" + "github.com/stretchr/testify/require" +) + +func TestNew(t *testing.T) { + tests := []struct { + name string + codec multicodec.Code + want Index + wantErr bool + }{ + { + name: "CarSortedIndexCodecIsConstructed", + codec: multicodec.CarIndexSorted, + want: newSorted(), + }, + { + name: "ValidMultiCodecButUnknwonToIndexIsError", + codec: multicodec.Cidv1, + wantErr: true, + }, + { + name: "IndexSingleSortedMultiCodecIsError", + codec: multicodec.Code(indexSingleSorted), + wantErr: true, + }, + { + name: "IndexHashedMultiCodecIsError", + codec: multicodec.Code(indexHashed), + wantErr: true, + }, + { + name: "IndexGobHashedMultiCodecIsError", + codec: multicodec.Code(indexGobHashed), + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := New(tt.codec) + if tt.wantErr { + require.Error(t, err) + } else { + require.Equal(t, tt.want, got) + } + }) + } +} + +func TestReadFrom(t *testing.T) { + idxf, err := os.Open("../testdata/sample-index.carindex") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, idxf.Close()) }) + + subject, err := ReadFrom(idxf) + require.NoError(t, err) + + crf, err := os.Open("../testdata/sample-v1.car") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, crf.Close()) }) + cr, err := carv1.NewCarReader(crf) + require.NoError(t, err) + + for { + wantBlock, err := cr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + + // Get offset from the index for a CID and assert it exists + gotOffset, err := subject.Get(wantBlock.Cid()) + require.NoError(t, err) + require.NotZero(t, gotOffset) + + // Seek to the offset on CARv1 file + _, err = crf.Seek(int64(gotOffset), io.SeekStart) + require.NoError(t, err) + + // Read the fame at offset and assert the frame corresponds to the expected block. + gotCid, gotData, err := util.ReadNode(crf) + require.NoError(t, err) + gotBlock, err := blocks.NewBlockWithCid(gotData, gotCid) + require.NoError(t, err) + require.Equal(t, wantBlock, gotBlock) + } +} + +func TestWriteTo(t *testing.T) { + // Read sample index on file + idxf, err := os.Open("../testdata/sample-index.carindex") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, idxf.Close()) }) + + // Unmarshall to get expected index + wantIdx, err := ReadFrom(idxf) + require.NoError(t, err) + + // Write the same index out + dest := filepath.Join(t.TempDir(), "index-write-to-test.carindex") + destF, err := os.Create(dest) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, destF.Close()) }) + require.NoError(t, WriteTo(wantIdx, destF)) + + // Seek to the beginning of the written out file. + _, err = destF.Seek(0, io.SeekStart) + require.NoError(t, err) + + // Read the written index back + gotIdx, err := ReadFrom(destF) + require.NoError(t, err) + + // Assert they are equal + require.Equal(t, wantIdx, gotIdx) +} + +func TestSave(t *testing.T) { + // Read sample index on file + idxf, err := os.Open("../testdata/sample-index.carindex") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, idxf.Close()) }) + + // Unmarshall to get expected index + wantIdx, err := ReadFrom(idxf) + require.NoError(t, err) + + // Save the same index at destination + dest := filepath.Join(t.TempDir(), "index-write-to-test.carindex") + require.NoError(t, Save(wantIdx, dest)) + + // Open the saved file + destF, err := os.Open(dest) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, destF.Close()) }) + + // Read the written index back + gotIdx, err := ReadFrom(destF) + require.NoError(t, err) + + // Assert they are equal + require.Equal(t, wantIdx, gotIdx) +} + +func TestMarshalledIndexStartsWithCodec(t *testing.T) { + // Read sample index on file + idxf, err := os.Open("../testdata/sample-index.carindex") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, idxf.Close()) }) + + // Unmarshall to get expected index + wantIdx, err := ReadFrom(idxf) + require.NoError(t, err) + + // Assert the first two bytes are the corresponding multicodec code. + buf := new(bytes.Buffer) + require.NoError(t, WriteTo(wantIdx, buf)) + require.Equal(t, varint.ToUvarint(uint64(multicodec.CarIndexSorted)), buf.Bytes()[:2]) +} diff --git a/v2/index/indexsorted_test.go b/v2/index/indexsorted_test.go index c491bedf5a02e8e9aa198c26e6e85496116d8246..fe0ca961a04bb5b49a1399ed318c9b995b80ee87 100644 --- a/v2/index/indexsorted_test.go +++ b/v2/index/indexsorted_test.go @@ -1,10 +1,11 @@ package index import ( + "testing" + "github.com/ipfs/go-merkledag" "github.com/multiformats/go-multicodec" "github.com/stretchr/testify/require" - "testing" ) func TestSortedIndexCodec(t *testing.T) { diff --git a/v2/testdata/sample-index.carindex b/v2/testdata/sample-index.carindex new file mode 100644 index 0000000000000000000000000000000000000000..0f91e36507fc21d3a93a00f32bf330e024405af8 Binary files /dev/null and b/v2/testdata/sample-index.carindex differ