merkledag_test.go 7.62 KB
Newer Older
1
package merkledag_test
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
2 3

import (
4
	"bytes"
5
	"errors"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
6
	"fmt"
7 8
	"io"
	"io/ioutil"
9
	"strings"
10
	"sync"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
11
	"testing"
12

13
	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
14
	key "github.com/ipfs/go-ipfs/blocks/key"
15
	bserv "github.com/ipfs/go-ipfs/blockservice"
16
	bstest "github.com/ipfs/go-ipfs/blockservice/test"
17 18 19 20
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	imp "github.com/ipfs/go-ipfs/importer"
	chunk "github.com/ipfs/go-ipfs/importer/chunk"
	. "github.com/ipfs/go-ipfs/merkledag"
21
	dstest "github.com/ipfs/go-ipfs/merkledag/test"
22 23
	"github.com/ipfs/go-ipfs/pin"
	uio "github.com/ipfs/go-ipfs/unixfs/io"
24 25
	ds "gx/ipfs/QmTxLSvdhwg68WJimdS6icLPhZi28aTp6b7uihC2Yb47Xk/go-datastore"
	dssync "gx/ipfs/QmTxLSvdhwg68WJimdS6icLPhZi28aTp6b7uihC2Yb47Xk/go-datastore/sync"
26
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
Jeromy's avatar
Jeromy committed
27
	"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
28 29
)

Jeromy's avatar
Jeromy committed
30 31
type dagservAndPinner struct {
	ds DAGService
32
	mp pin.Pinner
Jeromy's avatar
Jeromy committed
33 34 35
}

func getDagservAndPinner(t *testing.T) dagservAndPinner {
36 37
	db := dssync.MutexWrap(ds.NewMapDatastore())
	bs := bstore.NewBlockstore(db)
38
	blockserv := bserv.New(bs, offline.Exchange(bs))
Jeromy's avatar
Jeromy committed
39
	dserv := NewDAGService(blockserv)
40
	mpin := pin.NewPinner(db, dserv)
Jeromy's avatar
Jeromy committed
41 42 43 44 45 46
	return dagservAndPinner{
		ds: dserv,
		mp: mpin,
	}
}

Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
47 48
func TestNode(t *testing.T) {

49 50 51
	n1 := NodeWithData([]byte("beep"))
	n2 := NodeWithData([]byte("boop"))
	n3 := NodeWithData([]byte("beep boop"))
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
52 53 54 55 56 57 58 59 60
	if err := n3.AddNodeLink("beep-link", n1); err != nil {
		t.Error(err)
	}
	if err := n3.AddNodeLink("boop-link", n2); err != nil {
		t.Error(err)
	}

	printn := func(name string, n *Node) {
		fmt.Println(">", name)
61
		fmt.Println("data:", string(n.Data()))
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
62 63 64 65 66 67

		fmt.Println("links:")
		for _, l := range n.Links {
			fmt.Println("-", l.Name, l.Size, l.Hash)
		}

68
		e, err := n.EncodeProtobuf(false)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
69 70 71 72 73 74 75 76 77 78 79 80
		if err != nil {
			t.Error(err)
		} else {
			fmt.Println("encoded:", e)
		}

		h, err := n.Multihash()
		if err != nil {
			t.Error(err)
		} else {
			fmt.Println("hash:", h)
		}
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
81 82 83 84

		k, err := n.Key()
		if err != nil {
			t.Error(err)
85
		} else if k != key.Key(h) {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
86 87 88 89
			t.Error("Key is not equivalent to multihash")
		} else {
			fmt.Println("key: ", k)
		}
90 91

		SubtestNodeStat(t, n)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
92 93 94 95 96 97
	}

	printn("beep", n1)
	printn("boop", n2)
	printn("beep boop", n3)
}
98

99
func SubtestNodeStat(t *testing.T, n *Node) {
100
	enc, err := n.EncodeProtobuf(true)
101
	if err != nil {
102
		t.Error("n.EncodeProtobuf(true) failed")
103 104 105 106 107 108 109 110 111
		return
	}

	cumSize, err := n.Size()
	if err != nil {
		t.Error("n.Size() failed")
		return
	}

112 113 114 115 116 117
	k, err := n.Key()
	if err != nil {
		t.Error("n.Key() failed")
		return
	}

118 119 120
	expected := NodeStat{
		NumLinks:       len(n.Links),
		BlockSize:      len(enc),
121 122
		LinksSize:      len(enc) - len(n.Data()), // includes framing.
		DataSize:       len(n.Data()),
123
		CumulativeSize: int(cumSize),
124
		Hash:           k.B58String(),
125 126 127 128 129 130 131 132
	}

	actual, err := n.Stat()
	if err != nil {
		t.Error("n.Stat() failed")
		return
	}

133
	if expected != *actual {
134
		t.Errorf("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual)
135 136 137 138 139
	} else {
		fmt.Printf("n.Stat correct: %s\n", actual)
	}
}

140 141 142
type devZero struct{}

func (_ devZero) Read(b []byte) (int, error) {
rht's avatar
rht committed
143
	for i := range b {
144 145 146 147 148
		b[i] = 0
	}
	return len(b), nil
}

149
func TestBatchFetch(t *testing.T) {
Jeromy's avatar
Jeromy committed
150 151
	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
	runBatchFetchTest(t, read)
152
}
153 154

func TestBatchFetchDupBlock(t *testing.T) {
Jeromy's avatar
Jeromy committed
155 156
	read := io.LimitReader(devZero{}, 1024*32)
	runBatchFetchTest(t, read)
157 158
}

Jeromy's avatar
Jeromy committed
159
func runBatchFetchTest(t *testing.T, read io.Reader) {
160
	ctx := context.Background()
161
	var dagservs []DAGService
162
	for _, bsi := range bstest.Mocks(5) {
163 164
		dagservs = append(dagservs, NewDAGService(bsi))
	}
Jeromy's avatar
Jeromy committed
165

166
	spl := chunk.NewSizeSplitter(read, 512)
Jeromy's avatar
Jeromy committed
167

Jeromy's avatar
Jeromy committed
168
	root, err := imp.BuildDagFromReader(dagservs[0], spl)
Jeromy's avatar
Jeromy committed
169 170 171 172
	if err != nil {
		t.Fatal(err)
	}

173 174
	t.Log("finished setup.")

175
	dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
176 177 178
	if err != nil {
		t.Fatal(err)
	}
Jeromy's avatar
Jeromy committed
179 180

	expected, err := ioutil.ReadAll(dagr)
181 182 183 184
	if err != nil {
		t.Fatal(err)
	}

185
	_, err = dagservs[0].Add(root)
186 187 188 189 190 191 192 193 194 195 196
	if err != nil {
		t.Fatal(err)
	}

	t.Log("Added file to first node.")

	k, err := root.Key()
	if err != nil {
		t.Fatal(err)
	}

197
	wg := sync.WaitGroup{}
198 199
	errs := make(chan error)

200
	for i := 1; i < len(dagservs); i++ {
201
		wg.Add(1)
202
		go func(i int) {
203
			defer wg.Done()
204
			first, err := dagservs[i].Get(ctx, k)
205
			if err != nil {
206
				errs <- err
207 208 209
			}
			fmt.Println("Got first node back.")

210
			read, err := uio.NewDagReader(ctx, first, dagservs[i])
211
			if err != nil {
212
				errs <- err
213 214 215
			}
			datagot, err := ioutil.ReadAll(read)
			if err != nil {
216
				errs <- err
217 218 219
			}

			if !bytes.Equal(datagot, expected) {
220
				errs <- errors.New("Got bad data back!")
221 222 223 224
			}
		}(i)
	}

225 226 227 228 229 230 231 232 233 234
	go func() {
		wg.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
235
}
236 237 238 239 240 241 242

func assertCanGet(t *testing.T, ds DAGService, n *Node) {
	k, err := n.Key()
	if err != nil {
		t.Fatal(err)
	}

243
	if _, err := ds.Get(context.Background(), k); err != nil {
244 245 246 247
		t.Fatal(err)
	}
}

jbenet's avatar
jbenet committed
248
func TestEmptyKey(t *testing.T) {
Juan Benet's avatar
Juan Benet committed
249
	ds := dstest.Mock()
jbenet's avatar
jbenet committed
250 251 252 253 254 255
	_, err := ds.Get(context.Background(), key.Key(""))
	if err != ErrNotFound {
		t.Error("dag service should error when key is nil", err)
	}
}

256 257
func TestCantGet(t *testing.T) {
	dsp := getDagservAndPinner(t)
258
	a := NodeWithData([]byte("A"))
259 260 261 262 263 264

	k, err := a.Key()
	if err != nil {
		t.Fatal(err)
	}

265
	_, err = dsp.ds.Get(context.Background(), k)
266 267 268 269
	if !strings.Contains(err.Error(), "not found") {
		t.Fatal("expected err not found, got: ", err)
	}
}
270 271

func TestFetchGraph(t *testing.T) {
Jeromy's avatar
Jeromy committed
272
	var dservs []DAGService
273
	bsis := bstest.Mocks(2)
Jeromy's avatar
Jeromy committed
274 275 276
	for _, bsi := range bsis {
		dservs = append(dservs, NewDAGService(bsi))
	}
277 278

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
Jeromy's avatar
Jeromy committed
279
	root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
280 281 282 283
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
284
	err = FetchGraph(context.TODO(), root, dservs[1])
285 286 287 288
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
289
	// create an offline dagstore and ensure all blocks were fetched
290
	bs := bserv.New(bsis[1].Blockstore, offline.Exchange(bsis[1].Blockstore))
291

Jeromy's avatar
Jeromy committed
292 293 294
	offline_ds := NewDAGService(bs)
	ks := key.NewKeySet()

295
	err = EnumerateChildren(context.Background(), offline_ds, root, ks, false)
296 297 298 299 300 301
	if err != nil {
		t.Fatal(err)
	}
}

func TestEnumerateChildren(t *testing.T) {
302
	bsi := bstest.Mocks(1)
303 304 305
	ds := NewDAGService(bsi[0])

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
Jeromy's avatar
Jeromy committed
306
	root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
307 308 309 310 311
	if err != nil {
		t.Fatal(err)
	}

	ks := key.NewKeySet()
312
	err = EnumerateChildren(context.Background(), ds, root, ks, false)
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
	if err != nil {
		t.Fatal(err)
	}

	var traverse func(n *Node)
	traverse = func(n *Node) {
		// traverse dag and check
		for _, lnk := range n.Links {
			k := key.Key(lnk.Hash)
			if !ks.Has(k) {
				t.Fatal("missing key in set!")
			}
			child, err := ds.Get(context.Background(), k)
			if err != nil {
				t.Fatal(err)
			}
			traverse(child)
		}
	}

	traverse(root)
}
335 336 337 338 339 340 341

func TestFetchFailure(t *testing.T) {
	ds := dstest.Mock()
	ds_bad := dstest.Mock()

	top := new(Node)
	for i := 0; i < 10; i++ {
342
		nd := NodeWithData([]byte{byte('a' + i)})
343 344 345 346 347 348 349 350 351 352 353 354
		_, err := ds.Add(nd)
		if err != nil {
			t.Fatal(err)
		}

		err = top.AddNodeLinkClean(fmt.Sprintf("AA%d", i), nd)
		if err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < 10; i++ {
355
		nd := NodeWithData([]byte{'f', 'a' + byte(i)})
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
		_, err := ds_bad.Add(nd)
		if err != nil {
			t.Fatal(err)
		}

		err = top.AddNodeLinkClean(fmt.Sprintf("BB%d", i), nd)
		if err != nil {
			t.Fatal(err)
		}
	}

	getters := GetDAG(context.Background(), ds, top)
	for i, getter := range getters {
		_, err := getter.Get(context.Background())
		if err != nil && i < 10 {
			t.Fatal(err)
		}
		if err == nil && i >= 10 {
			t.Fatal("should have failed request")
		}
	}
}