merkledag_test.go 7.41 KB
Newer Older
1
package merkledag_test
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
2 3

import (
4
	"bytes"
5
	"errors"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
6
	"fmt"
7 8
	"io"
	"io/ioutil"
9
	"strings"
10
	"sync"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
11
	"testing"
12

13
	bserv "github.com/ipfs/go-ipfs/blockservice"
14
	bstest "github.com/ipfs/go-ipfs/blockservice/test"
15 16 17 18
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	imp "github.com/ipfs/go-ipfs/importer"
	chunk "github.com/ipfs/go-ipfs/importer/chunk"
	. "github.com/ipfs/go-ipfs/merkledag"
19
	mdpb "github.com/ipfs/go-ipfs/merkledag/pb"
20
	dstest "github.com/ipfs/go-ipfs/merkledag/test"
21
	uio "github.com/ipfs/go-ipfs/unixfs/io"
22
	key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
Jeromy's avatar
Jeromy committed
23

24 25 26
	"context"
	cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
27 28 29 30
)

func TestNode(t *testing.T) {

31 32 33
	n1 := NodeWithData([]byte("beep"))
	n2 := NodeWithData([]byte("boop"))
	n3 := NodeWithData([]byte("beep boop"))
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
34 35 36 37 38 39 40 41 42
	if err := n3.AddNodeLink("beep-link", n1); err != nil {
		t.Error(err)
	}
	if err := n3.AddNodeLink("boop-link", n2); err != nil {
		t.Error(err)
	}

	printn := func(name string, n *Node) {
		fmt.Println(">", name)
43
		fmt.Println("data:", string(n.Data()))
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
44 45 46 47 48 49

		fmt.Println("links:")
		for _, l := range n.Links {
			fmt.Println("-", l.Name, l.Size, l.Hash)
		}

50
		e, err := n.EncodeProtobuf(false)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
51 52 53 54 55 56
		if err != nil {
			t.Error(err)
		} else {
			fmt.Println("encoded:", e)
		}

Jeromy's avatar
Jeromy committed
57 58 59
		h := n.Multihash()
		k := n.Key()
		if k != key.Key(h) {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
60 61 62 63
			t.Error("Key is not equivalent to multihash")
		} else {
			fmt.Println("key: ", k)
		}
64 65

		SubtestNodeStat(t, n)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
66 67 68 69 70 71
	}

	printn("beep", n1)
	printn("boop", n2)
	printn("beep boop", n3)
}
72

73
func SubtestNodeStat(t *testing.T, n *Node) {
74
	enc, err := n.EncodeProtobuf(true)
75
	if err != nil {
76
		t.Error("n.EncodeProtobuf(true) failed")
77 78 79 80 81 82 83 84 85
		return
	}

	cumSize, err := n.Size()
	if err != nil {
		t.Error("n.Size() failed")
		return
	}

Jeromy's avatar
Jeromy committed
86
	k := n.Key()
87

88 89 90
	expected := NodeStat{
		NumLinks:       len(n.Links),
		BlockSize:      len(enc),
91 92
		LinksSize:      len(enc) - len(n.Data()), // includes framing.
		DataSize:       len(n.Data()),
93
		CumulativeSize: int(cumSize),
94
		Hash:           k.B58String(),
95 96 97 98 99 100 101 102
	}

	actual, err := n.Stat()
	if err != nil {
		t.Error("n.Stat() failed")
		return
	}

103
	if expected != *actual {
104
		t.Errorf("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual)
105 106 107 108 109
	} else {
		fmt.Printf("n.Stat correct: %s\n", actual)
	}
}

110 111 112
type devZero struct{}

func (_ devZero) Read(b []byte) (int, error) {
rht's avatar
rht committed
113
	for i := range b {
114 115 116 117 118
		b[i] = 0
	}
	return len(b), nil
}

119
func TestBatchFetch(t *testing.T) {
Jeromy's avatar
Jeromy committed
120 121
	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
	runBatchFetchTest(t, read)
122
}
123 124

func TestBatchFetchDupBlock(t *testing.T) {
Jeromy's avatar
Jeromy committed
125 126
	read := io.LimitReader(devZero{}, 1024*32)
	runBatchFetchTest(t, read)
127 128
}

Jeromy's avatar
Jeromy committed
129
func runBatchFetchTest(t *testing.T, read io.Reader) {
130
	ctx := context.Background()
131
	var dagservs []DAGService
132
	for _, bsi := range bstest.Mocks(5) {
133 134
		dagservs = append(dagservs, NewDAGService(bsi))
	}
Jeromy's avatar
Jeromy committed
135

136
	spl := chunk.NewSizeSplitter(read, 512)
Jeromy's avatar
Jeromy committed
137

Jeromy's avatar
Jeromy committed
138
	root, err := imp.BuildDagFromReader(dagservs[0], spl)
Jeromy's avatar
Jeromy committed
139 140 141 142
	if err != nil {
		t.Fatal(err)
	}

143 144
	t.Log("finished setup.")

145
	dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
146 147 148
	if err != nil {
		t.Fatal(err)
	}
Jeromy's avatar
Jeromy committed
149 150

	expected, err := ioutil.ReadAll(dagr)
151 152 153 154
	if err != nil {
		t.Fatal(err)
	}

155
	_, err = dagservs[0].Add(root)
156 157 158 159 160 161
	if err != nil {
		t.Fatal(err)
	}

	t.Log("Added file to first node.")

Jeromy's avatar
Jeromy committed
162
	c := root.Cid()
163

164
	wg := sync.WaitGroup{}
165 166
	errs := make(chan error)

167
	for i := 1; i < len(dagservs); i++ {
168
		wg.Add(1)
169
		go func(i int) {
170
			defer wg.Done()
Jeromy's avatar
Jeromy committed
171
			first, err := dagservs[i].Get(ctx, c)
172
			if err != nil {
173
				errs <- err
174 175 176
			}
			fmt.Println("Got first node back.")

177
			read, err := uio.NewDagReader(ctx, first, dagservs[i])
178
			if err != nil {
179
				errs <- err
180 181 182
			}
			datagot, err := ioutil.ReadAll(read)
			if err != nil {
183
				errs <- err
184 185 186
			}

			if !bytes.Equal(datagot, expected) {
187
				errs <- errors.New("Got bad data back!")
188 189 190 191
			}
		}(i)
	}

192 193 194 195 196 197 198 199 200 201
	go func() {
		wg.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
202
}
203 204

func assertCanGet(t *testing.T, ds DAGService, n *Node) {
Jeromy's avatar
Jeromy committed
205
	if _, err := ds.Get(context.Background(), n.Cid()); err != nil {
206 207 208 209 210
		t.Fatal(err)
	}
}

func TestCantGet(t *testing.T) {
211
	ds := dstest.Mock()
212
	a := NodeWithData([]byte("A"))
213

Jeromy's avatar
Jeromy committed
214 215
	c := a.Cid()
	_, err := ds.Get(context.Background(), c)
216 217 218 219
	if !strings.Contains(err.Error(), "not found") {
		t.Fatal("expected err not found, got: ", err)
	}
}
220 221

func TestFetchGraph(t *testing.T) {
Jeromy's avatar
Jeromy committed
222
	var dservs []DAGService
223
	bsis := bstest.Mocks(2)
Jeromy's avatar
Jeromy committed
224 225 226
	for _, bsi := range bsis {
		dservs = append(dservs, NewDAGService(bsi))
	}
227 228

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
Jeromy's avatar
Jeromy committed
229
	root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
230 231 232 233
	if err != nil {
		t.Fatal(err)
	}

234
	err = FetchGraph(context.TODO(), root.Cid(), dservs[1])
235 236 237 238
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
239
	// create an offline dagstore and ensure all blocks were fetched
240
	bs := bserv.New(bsis[1].Blockstore, offline.Exchange(bsis[1].Blockstore))
241

Jeromy's avatar
Jeromy committed
242 243
	offline_ds := NewDAGService(bs)

244
	err = EnumerateChildren(context.Background(), offline_ds, root.Cid(), func(_ *cid.Cid) bool { return true }, false)
245 246 247 248 249 250
	if err != nil {
		t.Fatal(err)
	}
}

func TestEnumerateChildren(t *testing.T) {
251
	bsi := bstest.Mocks(1)
252 253 254
	ds := NewDAGService(bsi[0])

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
Jeromy's avatar
Jeromy committed
255
	root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
256 257 258 259
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
260
	set := cid.NewSet()
261
	err = EnumerateChildren(context.Background(), ds, root.Cid(), set.Visit, false)
262 263 264 265 266 267 268 269
	if err != nil {
		t.Fatal(err)
	}

	var traverse func(n *Node)
	traverse = func(n *Node) {
		// traverse dag and check
		for _, lnk := range n.Links {
Jeromy's avatar
Jeromy committed
270 271
			c := cid.NewCidV0(lnk.Hash)
			if !set.Has(c) {
272
				t.Fatal("missing key in set! ", lnk.Hash.B58String())
273
			}
Jeromy's avatar
Jeromy committed
274
			child, err := ds.Get(context.Background(), c)
275 276 277 278 279 280 281 282 283
			if err != nil {
				t.Fatal(err)
			}
			traverse(child)
		}
	}

	traverse(root)
}
284 285 286 287 288 289 290

func TestFetchFailure(t *testing.T) {
	ds := dstest.Mock()
	ds_bad := dstest.Mock()

	top := new(Node)
	for i := 0; i < 10; i++ {
291
		nd := NodeWithData([]byte{byte('a' + i)})
292 293 294 295 296 297 298 299 300 301 302 303
		_, err := ds.Add(nd)
		if err != nil {
			t.Fatal(err)
		}

		err = top.AddNodeLinkClean(fmt.Sprintf("AA%d", i), nd)
		if err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < 10; i++ {
304
		nd := NodeWithData([]byte{'f', 'a' + byte(i)})
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326
		_, err := ds_bad.Add(nd)
		if err != nil {
			t.Fatal(err)
		}

		err = top.AddNodeLinkClean(fmt.Sprintf("BB%d", i), nd)
		if err != nil {
			t.Fatal(err)
		}
	}

	getters := GetDAG(context.Background(), ds, top)
	for i, getter := range getters {
		_, err := getter.Get(context.Background())
		if err != nil && i < 10 {
			t.Fatal(err)
		}
		if err == nil && i >= 10 {
			t.Fatal("should have failed request")
		}
	}
}
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350

func TestUnmarshalFailure(t *testing.T) {
	badData := []byte("hello world")

	_, err := DecodeProtobuf(badData)
	if err == nil {
		t.Fatal("shouldnt succeed to parse this")
	}

	// now with a bad link
	pbn := &mdpb.PBNode{Links: []*mdpb.PBLink{{Hash: []byte("not a multihash")}}}
	badlink, err := pbn.Marshal()
	if err != nil {
		t.Fatal(err)
	}

	_, err = DecodeProtobuf(badlink)
	if err == nil {
		t.Fatal("should have failed to parse node with bad link")
	}

	n := &Node{}
	n.Marshal()
}
Jeromy's avatar
Jeromy committed
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369

func TestBasicAddGet(t *testing.T) {
	ds := dstest.Mock()
	nd := new(Node)

	c, err := ds.Add(nd)
	if err != nil {
		t.Fatal(err)
	}

	out, err := ds.Get(context.Background(), c)
	if err != nil {
		t.Fatal(err)
	}

	if !nd.Cid().Equals(out.Cid()) {
		t.Fatal("output didnt match input")
	}
}