splitting_test.go

package chunk

import (
	"bytes"
	"io"
	"testing"

	u "github.com/ipfs/go-ipfs-util"
	util "github.com/ipfs/go-ipfs-util"
)

func randBuf(t *testing.T, size int) []byte {
	buf := make([]byte, size)
	if _, err := u.NewTimeSeededRand().Read(buf); err != nil {
		t.Fatal("failed to read enough randomness")
	}
	return buf
}

func copyBuf(buf []byte) []byte {
	cpy := make([]byte, len(buf))
	copy(cpy, buf)
	return cpy
}

func TestSizeSplitterOverAllocate(t *testing.T) {
	max := 1000
	r := bytes.NewReader(randBuf(t, max))
	chunksize := int64(1024 * 256)
	splitter := NewSizeSplitter(r, chunksize)
	chunk, err := splitter.NextBytes()
	if err != nil {
		t.Fatal(err)
	}
	if cap(chunk) > len(chunk) {
		t.Fatal("chunk capacity too large")
	}
}

func TestSizeSplitterIsDeterministic(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	test := func() {
		bufR := randBuf(t, 10000000) // crank this up to satisfy yourself.
		bufA := copyBuf(bufR)
		bufB := copyBuf(bufR)

		chunksA, _ := Chan(DefaultSplitter(bytes.NewReader(bufA)))
		chunksB, _ := Chan(DefaultSplitter(bytes.NewReader(bufB)))

		for n := 0; ; n++ {
			a, moreA := <-chunksA
			b, moreB := <-chunksB

			if !moreA {
				if moreB {
					t.Fatal("A ended, B didnt.")
				}
				return
			}

			if !bytes.Equal(a, b) {
				t.Fatalf("chunk %d not equal", n)
			}
		}
	}

	for run := 0; run < 1; run++ { // crank this up to satisfy yourself.
		test()
	}
}

func TestSizeSplitterFillsChunks(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	max := 10000000
	b := randBuf(t, max)
	r := &clipReader{r: bytes.NewReader(b), size: 4000}
	chunksize := int64(1024 * 256)
	c, _ := Chan(NewSizeSplitter(r, chunksize))

	sofar := 0
	whole := make([]byte, max)
	for chunk := range c {

		bc := b[sofar : sofar+len(chunk)]
		if !bytes.Equal(bc, chunk) {
			t.Fatalf("chunk not correct: (sofar: %d) %d != %d, %v != %v", sofar, len(bc), len(chunk), bc[:100], chunk[:100])
		}

		copy(whole[sofar:], chunk)

		sofar += len(chunk)
		if sofar != max && len(chunk) < int(chunksize) {
			t.Fatal("sizesplitter split at a smaller size")
		}
	}

	if !bytes.Equal(b, whole) {
		t.Fatal("splitter did not split right")
	}
}

type clipReader struct {
	size int
	r    io.Reader
}

func (s *clipReader) Read(buf []byte) (int, error) {

	// clip the incoming buffer to produce smaller chunks
	if len(buf) > s.size {
		buf = buf[:s.size]
	}

	return s.r.Read(buf)
}

func BenchmarkDefault(b *testing.B) {
	data := make([]byte, 16<<20)
	util.NewTimeSeededRand().Read(data)

	b.SetBytes(16 << 20)
	b.ReportAllocs()
	b.ResetTimer()

	var res uint64

	for i := 0; i < b.N; i++ {
		r := DefaultSplitter(bytes.NewReader(data))

		for {
			chunk, err := r.NextBytes()
			if err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
			res = res + uint64(len(chunk))
		}
	}
	Res = Res + res
}