merkledag_test.go 6.55 KB
Newer Older
1
package merkledag_test
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
2 3

import (
4
	"bytes"
5
	"errors"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
6
	"fmt"
7 8
	"io"
	"io/ioutil"
9
	"strings"
10
	"sync"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
11
	"testing"
12

Jeromy's avatar
Jeromy committed
13 14
	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	dssync "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/sync"
15
	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
16
	key "github.com/ipfs/go-ipfs/blocks/key"
17
	bserv "github.com/ipfs/go-ipfs/blockservice"
18
	bstest "github.com/ipfs/go-ipfs/blockservice/test"
19 20 21 22 23 24
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	imp "github.com/ipfs/go-ipfs/importer"
	chunk "github.com/ipfs/go-ipfs/importer/chunk"
	. "github.com/ipfs/go-ipfs/merkledag"
	"github.com/ipfs/go-ipfs/pin"
	uio "github.com/ipfs/go-ipfs/unixfs/io"
25
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
Jeromy's avatar
Jeromy committed
26
	"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
27 28
)

Jeromy's avatar
Jeromy committed
29 30
type dagservAndPinner struct {
	ds DAGService
31
	mp pin.Pinner
Jeromy's avatar
Jeromy committed
32 33 34
}

func getDagservAndPinner(t *testing.T) dagservAndPinner {
35 36
	db := dssync.MutexWrap(ds.NewMapDatastore())
	bs := bstore.NewBlockstore(db)
37
	blockserv := bserv.New(bs, offline.Exchange(bs))
Jeromy's avatar
Jeromy committed
38
	dserv := NewDAGService(blockserv)
39
	mpin := pin.NewPinner(db, dserv)
Jeromy's avatar
Jeromy committed
40 41 42 43 44 45
	return dagservAndPinner{
		ds: dserv,
		mp: mpin,
	}
}

Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
func TestNode(t *testing.T) {

	n1 := &Node{Data: []byte("beep")}
	n2 := &Node{Data: []byte("boop")}
	n3 := &Node{Data: []byte("beep boop")}
	if err := n3.AddNodeLink("beep-link", n1); err != nil {
		t.Error(err)
	}
	if err := n3.AddNodeLink("boop-link", n2); err != nil {
		t.Error(err)
	}

	printn := func(name string, n *Node) {
		fmt.Println(">", name)
		fmt.Println("data:", string(n.Data))

		fmt.Println("links:")
		for _, l := range n.Links {
			fmt.Println("-", l.Name, l.Size, l.Hash)
		}

67
		e, err := n.EncodeProtobuf(false)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
68 69 70 71 72 73 74 75 76 77 78 79
		if err != nil {
			t.Error(err)
		} else {
			fmt.Println("encoded:", e)
		}

		h, err := n.Multihash()
		if err != nil {
			t.Error(err)
		} else {
			fmt.Println("hash:", h)
		}
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
80 81 82 83

		k, err := n.Key()
		if err != nil {
			t.Error(err)
84
		} else if k != key.Key(h) {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
85 86 87 88
			t.Error("Key is not equivalent to multihash")
		} else {
			fmt.Println("key: ", k)
		}
89 90

		SubtestNodeStat(t, n)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
91 92 93 94 95 96
	}

	printn("beep", n1)
	printn("boop", n2)
	printn("beep boop", n3)
}
97

98
func SubtestNodeStat(t *testing.T, n *Node) {
99
	enc, err := n.EncodeProtobuf(true)
100
	if err != nil {
101
		t.Error("n.EncodeProtobuf(true) failed")
102 103 104 105 106 107 108 109 110
		return
	}

	cumSize, err := n.Size()
	if err != nil {
		t.Error("n.Size() failed")
		return
	}

111 112 113 114 115 116
	k, err := n.Key()
	if err != nil {
		t.Error("n.Key() failed")
		return
	}

117 118 119 120 121 122
	expected := NodeStat{
		NumLinks:       len(n.Links),
		BlockSize:      len(enc),
		LinksSize:      len(enc) - len(n.Data), // includes framing.
		DataSize:       len(n.Data),
		CumulativeSize: int(cumSize),
123
		Hash:           k.B58String(),
124 125 126 127 128 129 130 131
	}

	actual, err := n.Stat()
	if err != nil {
		t.Error("n.Stat() failed")
		return
	}

132
	if expected != *actual {
133
		t.Error("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual)
134 135 136 137 138
	} else {
		fmt.Printf("n.Stat correct: %s\n", actual)
	}
}

139 140 141
type devZero struct{}

func (_ devZero) Read(b []byte) (int, error) {
rht's avatar
rht committed
142
	for i := range b {
143 144 145 146 147
		b[i] = 0
	}
	return len(b), nil
}

148
func TestBatchFetch(t *testing.T) {
Jeromy's avatar
Jeromy committed
149 150
	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
	runBatchFetchTest(t, read)
151
}
152 153

func TestBatchFetchDupBlock(t *testing.T) {
Jeromy's avatar
Jeromy committed
154 155
	read := io.LimitReader(devZero{}, 1024*32)
	runBatchFetchTest(t, read)
156 157
}

Jeromy's avatar
Jeromy committed
158
func runBatchFetchTest(t *testing.T, read io.Reader) {
159
	ctx := context.Background()
160
	var dagservs []DAGService
161
	for _, bsi := range bstest.Mocks(5) {
162 163
		dagservs = append(dagservs, NewDAGService(bsi))
	}
Jeromy's avatar
Jeromy committed
164

165
	spl := chunk.NewSizeSplitter(read, 512)
Jeromy's avatar
Jeromy committed
166

Jeromy's avatar
Jeromy committed
167
	root, err := imp.BuildDagFromReader(dagservs[0], spl)
Jeromy's avatar
Jeromy committed
168 169 170 171
	if err != nil {
		t.Fatal(err)
	}

172 173
	t.Log("finished setup.")

174
	dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
175 176 177
	if err != nil {
		t.Fatal(err)
	}
Jeromy's avatar
Jeromy committed
178 179

	expected, err := ioutil.ReadAll(dagr)
180 181 182 183
	if err != nil {
		t.Fatal(err)
	}

184
	_, err = dagservs[0].Add(root)
185 186 187 188 189 190 191 192 193 194 195
	if err != nil {
		t.Fatal(err)
	}

	t.Log("Added file to first node.")

	k, err := root.Key()
	if err != nil {
		t.Fatal(err)
	}

196
	wg := sync.WaitGroup{}
197 198
	errs := make(chan error)

199
	for i := 1; i < len(dagservs); i++ {
200
		wg.Add(1)
201
		go func(i int) {
202
			defer wg.Done()
203
			first, err := dagservs[i].Get(ctx, k)
204
			if err != nil {
205
				errs <- err
206 207 208
			}
			fmt.Println("Got first node back.")

209
			read, err := uio.NewDagReader(ctx, first, dagservs[i])
210
			if err != nil {
211
				errs <- err
212 213 214
			}
			datagot, err := ioutil.ReadAll(read)
			if err != nil {
215
				errs <- err
216 217 218
			}

			if !bytes.Equal(datagot, expected) {
219
				errs <- errors.New("Got bad data back!")
220 221 222 223
			}
		}(i)
	}

224 225 226 227 228 229 230 231 232 233
	go func() {
		wg.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
234
}
235 236 237 238 239 240 241

func assertCanGet(t *testing.T, ds DAGService, n *Node) {
	k, err := n.Key()
	if err != nil {
		t.Fatal(err)
	}

242
	if _, err := ds.Get(context.Background(), k); err != nil {
243 244 245 246 247 248 249 250 251 252 253 254 255
		t.Fatal(err)
	}
}

func TestCantGet(t *testing.T) {
	dsp := getDagservAndPinner(t)
	a := &Node{Data: []byte("A")}

	k, err := a.Key()
	if err != nil {
		t.Fatal(err)
	}

256
	_, err = dsp.ds.Get(context.Background(), k)
257 258 259 260
	if !strings.Contains(err.Error(), "not found") {
		t.Fatal("expected err not found, got: ", err)
	}
}
261 262

func TestFetchGraph(t *testing.T) {
Jeromy's avatar
Jeromy committed
263
	var dservs []DAGService
264
	bsis := bstest.Mocks(2)
Jeromy's avatar
Jeromy committed
265 266 267
	for _, bsi := range bsis {
		dservs = append(dservs, NewDAGService(bsi))
	}
268 269

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
Jeromy's avatar
Jeromy committed
270
	root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
271 272 273 274
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
275
	err = FetchGraph(context.TODO(), root, dservs[1])
276 277 278 279
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
280
	// create an offline dagstore and ensure all blocks were fetched
281
	bs := bserv.New(bsis[1].Blockstore, offline.Exchange(bsis[1].Blockstore))
282

Jeromy's avatar
Jeromy committed
283 284 285 286
	offline_ds := NewDAGService(bs)
	ks := key.NewKeySet()

	err = EnumerateChildren(context.Background(), offline_ds, root, ks)
287 288 289 290 291 292
	if err != nil {
		t.Fatal(err)
	}
}

func TestEnumerateChildren(t *testing.T) {
293
	bsi := bstest.Mocks(1)
294 295 296
	ds := NewDAGService(bsi[0])

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
Jeromy's avatar
Jeromy committed
297
	root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
	if err != nil {
		t.Fatal(err)
	}

	ks := key.NewKeySet()
	err = EnumerateChildren(context.Background(), ds, root, ks)
	if err != nil {
		t.Fatal(err)
	}

	var traverse func(n *Node)
	traverse = func(n *Node) {
		// traverse dag and check
		for _, lnk := range n.Links {
			k := key.Key(lnk.Hash)
			if !ks.Has(k) {
				t.Fatal("missing key in set!")
			}
			child, err := ds.Get(context.Background(), k)
			if err != nil {
				t.Fatal(err)
			}
			traverse(child)
		}
	}

	traverse(root)
}