merkledag_test.go 7.59 KB
Newer Older
1
package merkledag_test
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
2 3

import (
4
	"bytes"
Jeromy's avatar
Jeromy committed
5
	"context"
6
	"errors"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
7
	"fmt"
8 9
	"io"
	"io/ioutil"
10
	"strings"
11
	"sync"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
12
	"testing"
13

14
	bserv "github.com/ipfs/go-ipfs/blockservice"
15
	bstest "github.com/ipfs/go-ipfs/blockservice/test"
16 17 18 19
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	imp "github.com/ipfs/go-ipfs/importer"
	chunk "github.com/ipfs/go-ipfs/importer/chunk"
	. "github.com/ipfs/go-ipfs/merkledag"
20
	mdpb "github.com/ipfs/go-ipfs/merkledag/pb"
21
	dstest "github.com/ipfs/go-ipfs/merkledag/test"
22
	uio "github.com/ipfs/go-ipfs/unixfs/io"
Jeromy's avatar
Jeromy committed
23

24
	cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
Jeromy's avatar
Jeromy committed
25 26
	key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
27
	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
28 29 30 31
)

func TestNode(t *testing.T) {

32 33 34
	n1 := NodeWithData([]byte("beep"))
	n2 := NodeWithData([]byte("boop"))
	n3 := NodeWithData([]byte("beep boop"))
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
35 36 37 38 39 40 41
	if err := n3.AddNodeLink("beep-link", n1); err != nil {
		t.Error(err)
	}
	if err := n3.AddNodeLink("boop-link", n2); err != nil {
		t.Error(err)
	}

42
	printn := func(name string, n *ProtoNode) {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
43
		fmt.Println(">", name)
44
		fmt.Println("data:", string(n.Data()))
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
45 46

		fmt.Println("links:")
47 48
		for _, l := range n.Links() {
			fmt.Println("-", l.Name, l.Size, l.Cid)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
49 50
		}

51
		e, err := n.EncodeProtobuf(false)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
52 53 54 55 56 57
		if err != nil {
			t.Error(err)
		} else {
			fmt.Println("encoded:", e)
		}

Jeromy's avatar
Jeromy committed
58 59 60
		h := n.Multihash()
		k := n.Key()
		if k != key.Key(h) {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
61 62 63 64
			t.Error("Key is not equivalent to multihash")
		} else {
			fmt.Println("key: ", k)
		}
65 66

		SubtestNodeStat(t, n)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
67 68 69 70 71 72
	}

	printn("beep", n1)
	printn("boop", n2)
	printn("beep boop", n3)
}
73

74
func SubtestNodeStat(t *testing.T, n *ProtoNode) {
75
	enc, err := n.EncodeProtobuf(true)
76
	if err != nil {
77
		t.Error("n.EncodeProtobuf(true) failed")
78 79 80 81 82 83 84 85 86
		return
	}

	cumSize, err := n.Size()
	if err != nil {
		t.Error("n.Size() failed")
		return
	}

Jeromy's avatar
Jeromy committed
87
	k := n.Key()
88

Jeromy's avatar
Jeromy committed
89
	expected := node.NodeStat{
90
		NumLinks:       len(n.Links()),
91
		BlockSize:      len(enc),
92 93
		LinksSize:      len(enc) - len(n.Data()), // includes framing.
		DataSize:       len(n.Data()),
94
		CumulativeSize: int(cumSize),
95
		Hash:           k.B58String(),
96 97 98 99 100 101 102 103
	}

	actual, err := n.Stat()
	if err != nil {
		t.Error("n.Stat() failed")
		return
	}

104
	if expected != *actual {
105
		t.Errorf("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual)
106 107 108 109 110
	} else {
		fmt.Printf("n.Stat correct: %s\n", actual)
	}
}

111 112 113
type devZero struct{}

func (_ devZero) Read(b []byte) (int, error) {
rht's avatar
rht committed
114
	for i := range b {
115 116 117 118 119
		b[i] = 0
	}
	return len(b), nil
}

120
func TestBatchFetch(t *testing.T) {
Jeromy's avatar
Jeromy committed
121 122
	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
	runBatchFetchTest(t, read)
123
}
124 125

func TestBatchFetchDupBlock(t *testing.T) {
Jeromy's avatar
Jeromy committed
126 127
	read := io.LimitReader(devZero{}, 1024*32)
	runBatchFetchTest(t, read)
128 129
}

Jeromy's avatar
Jeromy committed
130
func runBatchFetchTest(t *testing.T, read io.Reader) {
131
	ctx := context.Background()
132
	var dagservs []DAGService
133
	for _, bsi := range bstest.Mocks(5) {
134 135
		dagservs = append(dagservs, NewDAGService(bsi))
	}
Jeromy's avatar
Jeromy committed
136

137
	spl := chunk.NewSizeSplitter(read, 512)
Jeromy's avatar
Jeromy committed
138

Jeromy's avatar
Jeromy committed
139
	root, err := imp.BuildDagFromReader(dagservs[0], spl)
Jeromy's avatar
Jeromy committed
140 141 142 143
	if err != nil {
		t.Fatal(err)
	}

144 145
	t.Log("finished setup.")

146
	dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
147 148 149
	if err != nil {
		t.Fatal(err)
	}
Jeromy's avatar
Jeromy committed
150 151

	expected, err := ioutil.ReadAll(dagr)
152 153 154 155
	if err != nil {
		t.Fatal(err)
	}

156
	_, err = dagservs[0].Add(root)
157 158 159 160 161 162
	if err != nil {
		t.Fatal(err)
	}

	t.Log("Added file to first node.")

Jeromy's avatar
Jeromy committed
163
	c := root.Cid()
164

165
	wg := sync.WaitGroup{}
166 167
	errs := make(chan error)

168
	for i := 1; i < len(dagservs); i++ {
169
		wg.Add(1)
170
		go func(i int) {
171
			defer wg.Done()
Jeromy's avatar
Jeromy committed
172
			first, err := dagservs[i].Get(ctx, c)
173
			if err != nil {
174
				errs <- err
175 176 177
			}
			fmt.Println("Got first node back.")

178 179 180 181 182 183
			firstpb, ok := first.(*ProtoNode)
			if !ok {
				errs <- ErrNotProtobuf
			}

			read, err := uio.NewDagReader(ctx, firstpb, dagservs[i])
184
			if err != nil {
185
				errs <- err
186 187 188
			}
			datagot, err := ioutil.ReadAll(read)
			if err != nil {
189
				errs <- err
190 191 192
			}

			if !bytes.Equal(datagot, expected) {
193
				errs <- errors.New("Got bad data back!")
194 195 196 197
			}
		}(i)
	}

198 199 200 201 202 203 204 205 206 207
	go func() {
		wg.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
208
}
209

Jeromy's avatar
Jeromy committed
210
func assertCanGet(t *testing.T, ds DAGService, n node.Node) {
Jeromy's avatar
Jeromy committed
211
	if _, err := ds.Get(context.Background(), n.Cid()); err != nil {
212 213 214 215 216
		t.Fatal(err)
	}
}

func TestCantGet(t *testing.T) {
217
	ds := dstest.Mock()
218
	a := NodeWithData([]byte("A"))
219

Jeromy's avatar
Jeromy committed
220 221
	c := a.Cid()
	_, err := ds.Get(context.Background(), c)
222 223 224 225
	if !strings.Contains(err.Error(), "not found") {
		t.Fatal("expected err not found, got: ", err)
	}
}
226 227

func TestFetchGraph(t *testing.T) {
Jeromy's avatar
Jeromy committed
228
	var dservs []DAGService
229
	bsis := bstest.Mocks(2)
Jeromy's avatar
Jeromy committed
230 231 232
	for _, bsi := range bsis {
		dservs = append(dservs, NewDAGService(bsi))
	}
233 234

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
Jeromy's avatar
Jeromy committed
235
	root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
236 237 238 239
	if err != nil {
		t.Fatal(err)
	}

240
	err = FetchGraph(context.TODO(), root.Cid(), dservs[1])
241 242 243 244
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
245
	// create an offline dagstore and ensure all blocks were fetched
246
	bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore()))
247

Jeromy's avatar
Jeromy committed
248 249
	offline_ds := NewDAGService(bs)

250
	err = EnumerateChildren(context.Background(), offline_ds, root.Cid(), func(_ *cid.Cid) bool { return true }, false)
251 252 253 254 255 256
	if err != nil {
		t.Fatal(err)
	}
}

func TestEnumerateChildren(t *testing.T) {
257
	bsi := bstest.Mocks(1)
258 259 260
	ds := NewDAGService(bsi[0])

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
Jeromy's avatar
Jeromy committed
261
	root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
262 263 264 265
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
266
	set := cid.NewSet()
267
	err = EnumerateChildren(context.Background(), ds, root.Cid(), set.Visit, false)
268 269 270 271
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
272 273
	var traverse func(n node.Node)
	traverse = func(n node.Node) {
274
		// traverse dag and check
275 276
		for _, lnk := range n.Links() {
			c := lnk.Cid
Jeromy's avatar
Jeromy committed
277
			if !set.Has(c) {
278
				t.Fatal("missing key in set! ", lnk.Cid.String())
279
			}
Jeromy's avatar
Jeromy committed
280
			child, err := ds.Get(context.Background(), c)
281 282 283 284 285 286 287 288 289
			if err != nil {
				t.Fatal(err)
			}
			traverse(child)
		}
	}

	traverse(root)
}
290 291 292 293 294

func TestFetchFailure(t *testing.T) {
	ds := dstest.Mock()
	ds_bad := dstest.Mock()

295
	top := new(ProtoNode)
296
	for i := 0; i < 10; i++ {
297
		nd := NodeWithData([]byte{byte('a' + i)})
298 299 300 301 302 303 304 305 306 307 308 309
		_, err := ds.Add(nd)
		if err != nil {
			t.Fatal(err)
		}

		err = top.AddNodeLinkClean(fmt.Sprintf("AA%d", i), nd)
		if err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < 10; i++ {
310
		nd := NodeWithData([]byte{'f', 'a' + byte(i)})
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
		_, err := ds_bad.Add(nd)
		if err != nil {
			t.Fatal(err)
		}

		err = top.AddNodeLinkClean(fmt.Sprintf("BB%d", i), nd)
		if err != nil {
			t.Fatal(err)
		}
	}

	getters := GetDAG(context.Background(), ds, top)
	for i, getter := range getters {
		_, err := getter.Get(context.Background())
		if err != nil && i < 10 {
			t.Fatal(err)
		}
		if err == nil && i >= 10 {
			t.Fatal("should have failed request")
		}
	}
}
333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353

func TestUnmarshalFailure(t *testing.T) {
	badData := []byte("hello world")

	_, err := DecodeProtobuf(badData)
	if err == nil {
		t.Fatal("shouldnt succeed to parse this")
	}

	// now with a bad link
	pbn := &mdpb.PBNode{Links: []*mdpb.PBLink{{Hash: []byte("not a multihash")}}}
	badlink, err := pbn.Marshal()
	if err != nil {
		t.Fatal(err)
	}

	_, err = DecodeProtobuf(badlink)
	if err == nil {
		t.Fatal("should have failed to parse node with bad link")
	}

354
	n := &ProtoNode{}
355 356
	n.Marshal()
}
Jeromy's avatar
Jeromy committed
357 358 359

func TestBasicAddGet(t *testing.T) {
	ds := dstest.Mock()
360
	nd := new(ProtoNode)
Jeromy's avatar
Jeromy committed
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375

	c, err := ds.Add(nd)
	if err != nil {
		t.Fatal(err)
	}

	out, err := ds.Get(context.Background(), c)
	if err != nil {
		t.Fatal(err)
	}

	if !nd.Cid().Equals(out.Cid()) {
		t.Fatal("output didnt match input")
	}
}