package chunk import ( "bytes" "io" "testing" u "github.com/ipfs/go-ipfs-util" util "github.com/ipfs/go-ipfs-util" pool "github.com/libp2p/go-buffer-pool" ) func randBuf(t *testing.T, size int) []byte { buf := make([]byte, size) if _, err := u.NewTimeSeededRand().Read(buf); err != nil { t.Fatal("failed to read enough randomness") } return buf } func copyBuf(buf []byte) []byte { cpy := make([]byte, len(buf)) copy(cpy, buf) return cpy } func TestSizeSplitterOverAllocate(t *testing.T) { max := 1000 r := bytes.NewReader(randBuf(t, max)) chunksize := int64(1024 * 256) splitter := NewSizeSplitter(r, chunksize) chunk, err := splitter.NextBytes() if err != nil { t.Fatal(err) } if cap(chunk) > len(chunk) { t.Fatal("chunk capacity too large") } } func TestSizeSplitterIsDeterministic(t *testing.T) { if testing.Short() { t.SkipNow() } test := func() { bufR := randBuf(t, 10000000) // crank this up to satisfy yourself. bufA := copyBuf(bufR) bufB := copyBuf(bufR) chunksA, _ := Chan(DefaultSplitter(bytes.NewReader(bufA))) chunksB, _ := Chan(DefaultSplitter(bytes.NewReader(bufB))) for n := 0; ; n++ { a, moreA := <-chunksA b, moreB := <-chunksB if !moreA { if moreB { t.Fatal("A ended, B didnt.") } return } if !bytes.Equal(a, b) { t.Fatalf("chunk %d not equal", n) } } } for run := 0; run < 1; run++ { // crank this up to satisfy yourself. test() } } func TestSizeSplitterFillsChunks(t *testing.T) { if testing.Short() { t.SkipNow() } max := 10000000 b := randBuf(t, max) r := &clipReader{r: bytes.NewReader(b), size: 4000} chunksize := int64(1024 * 256) c, _ := Chan(NewSizeSplitter(r, chunksize)) sofar := 0 whole := make([]byte, max) for chunk := range c { bc := b[sofar : sofar+len(chunk)] if !bytes.Equal(bc, chunk) { t.Fatalf("chunk not correct: (sofar: %d) %d != %d, %v != %v", sofar, len(bc), len(chunk), bc[:100], chunk[:100]) } copy(whole[sofar:], chunk) sofar += len(chunk) if sofar != max && len(chunk) < int(chunksize) { t.Fatal("sizesplitter split at a smaller size") } } if !bytes.Equal(b, whole) { t.Fatal("splitter did not split right") } } type clipReader struct { size int r io.Reader } func (s *clipReader) Read(buf []byte) (int, error) { // clip the incoming buffer to produce smaller chunks if len(buf) > s.size { buf = buf[:s.size] } return s.r.Read(buf) } func BenchmarkDefault(b *testing.B) { data := make([]byte, 16<<20) util.NewTimeSeededRand().Read(data) b.SetBytes(16 << 20) b.ReportAllocs() b.ResetTimer() var res uint64 for i := 0; i < b.N; i++ { r := DefaultSplitter(bytes.NewReader(data)) for { chunk, err := r.NextBytes() if err != nil { if err == io.EOF { break } b.Fatal(err) } res = res + uint64(len(chunk)) pool.Put(chunk) } } Res = Res + res }