merkledag_test.go 7.5 KB
Newer Older
1
package merkledag_test
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
2 3

import (
4
	"bytes"
5
	"errors"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
6
	"fmt"
7 8
	"io"
	"io/ioutil"
9
	"strings"
10
	"sync"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
11
	"testing"
12

13
	bserv "github.com/ipfs/go-ipfs/blockservice"
14
	bstest "github.com/ipfs/go-ipfs/blockservice/test"
15 16 17 18
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	imp "github.com/ipfs/go-ipfs/importer"
	chunk "github.com/ipfs/go-ipfs/importer/chunk"
	. "github.com/ipfs/go-ipfs/merkledag"
19
	mdpb "github.com/ipfs/go-ipfs/merkledag/pb"
20
	dstest "github.com/ipfs/go-ipfs/merkledag/test"
21
	uio "github.com/ipfs/go-ipfs/unixfs/io"
22
	key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
Jeromy's avatar
Jeromy committed
23

24
	"context"
25
	cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
26
	u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
27 28 29 30
)

func TestNode(t *testing.T) {

31 32 33
	n1 := NodeWithData([]byte("beep"))
	n2 := NodeWithData([]byte("boop"))
	n3 := NodeWithData([]byte("beep boop"))
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
34 35 36 37 38 39 40
	if err := n3.AddNodeLink("beep-link", n1); err != nil {
		t.Error(err)
	}
	if err := n3.AddNodeLink("boop-link", n2); err != nil {
		t.Error(err)
	}

41
	printn := func(name string, n *ProtoNode) {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
42
		fmt.Println(">", name)
43
		fmt.Println("data:", string(n.Data()))
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
44 45

		fmt.Println("links:")
46 47
		for _, l := range n.Links() {
			fmt.Println("-", l.Name, l.Size, l.Cid)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
48 49
		}

50
		e, err := n.EncodeProtobuf(false)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
51 52 53 54 55 56
		if err != nil {
			t.Error(err)
		} else {
			fmt.Println("encoded:", e)
		}

Jeromy's avatar
Jeromy committed
57 58 59
		h := n.Multihash()
		k := n.Key()
		if k != key.Key(h) {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
60 61 62 63
			t.Error("Key is not equivalent to multihash")
		} else {
			fmt.Println("key: ", k)
		}
64 65

		SubtestNodeStat(t, n)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
66 67 68 69 70 71
	}

	printn("beep", n1)
	printn("boop", n2)
	printn("beep boop", n3)
}
72

73
func SubtestNodeStat(t *testing.T, n *ProtoNode) {
74
	enc, err := n.EncodeProtobuf(true)
75
	if err != nil {
76
		t.Error("n.EncodeProtobuf(true) failed")
77 78 79 80 81 82 83 84 85
		return
	}

	cumSize, err := n.Size()
	if err != nil {
		t.Error("n.Size() failed")
		return
	}

Jeromy's avatar
Jeromy committed
86
	k := n.Key()
87

88
	expected := NodeStat{
89
		NumLinks:       len(n.Links()),
90
		BlockSize:      len(enc),
91 92
		LinksSize:      len(enc) - len(n.Data()), // includes framing.
		DataSize:       len(n.Data()),
93
		CumulativeSize: int(cumSize),
94
		Hash:           k.B58String(),
95 96 97 98 99 100 101 102
	}

	actual, err := n.Stat()
	if err != nil {
		t.Error("n.Stat() failed")
		return
	}

103
	if expected != *actual {
104
		t.Errorf("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual)
105 106 107 108 109
	} else {
		fmt.Printf("n.Stat correct: %s\n", actual)
	}
}

110 111 112
type devZero struct{}

func (_ devZero) Read(b []byte) (int, error) {
rht's avatar
rht committed
113
	for i := range b {
114 115 116 117 118
		b[i] = 0
	}
	return len(b), nil
}

119
func TestBatchFetch(t *testing.T) {
Jeromy's avatar
Jeromy committed
120 121
	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
	runBatchFetchTest(t, read)
122
}
123 124

func TestBatchFetchDupBlock(t *testing.T) {
Jeromy's avatar
Jeromy committed
125 126
	read := io.LimitReader(devZero{}, 1024*32)
	runBatchFetchTest(t, read)
127 128
}

Jeromy's avatar
Jeromy committed
129
func runBatchFetchTest(t *testing.T, read io.Reader) {
130
	ctx := context.Background()
131
	var dagservs []DAGService
132
	for _, bsi := range bstest.Mocks(5) {
133 134
		dagservs = append(dagservs, NewDAGService(bsi))
	}
Jeromy's avatar
Jeromy committed
135

136
	spl := chunk.NewSizeSplitter(read, 512)
Jeromy's avatar
Jeromy committed
137

Jeromy's avatar
Jeromy committed
138
	root, err := imp.BuildDagFromReader(dagservs[0], spl)
Jeromy's avatar
Jeromy committed
139 140 141 142
	if err != nil {
		t.Fatal(err)
	}

143 144
	t.Log("finished setup.")

145
	dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
146 147 148
	if err != nil {
		t.Fatal(err)
	}
Jeromy's avatar
Jeromy committed
149 150

	expected, err := ioutil.ReadAll(dagr)
151 152 153 154
	if err != nil {
		t.Fatal(err)
	}

155
	_, err = dagservs[0].Add(root)
156 157 158 159 160 161
	if err != nil {
		t.Fatal(err)
	}

	t.Log("Added file to first node.")

Jeromy's avatar
Jeromy committed
162
	c := root.Cid()
163

164
	wg := sync.WaitGroup{}
165 166
	errs := make(chan error)

167
	for i := 1; i < len(dagservs); i++ {
168
		wg.Add(1)
169
		go func(i int) {
170
			defer wg.Done()
Jeromy's avatar
Jeromy committed
171
			first, err := dagservs[i].Get(ctx, c)
172
			if err != nil {
173
				errs <- err
174 175 176
			}
			fmt.Println("Got first node back.")

177 178 179 180 181 182
			firstpb, ok := first.(*ProtoNode)
			if !ok {
				errs <- ErrNotProtobuf
			}

			read, err := uio.NewDagReader(ctx, firstpb, dagservs[i])
183
			if err != nil {
184
				errs <- err
185 186 187
			}
			datagot, err := ioutil.ReadAll(read)
			if err != nil {
188
				errs <- err
189 190 191
			}

			if !bytes.Equal(datagot, expected) {
192
				errs <- errors.New("Got bad data back!")
193 194 195 196
			}
		}(i)
	}

197 198 199 200 201 202 203 204 205 206
	go func() {
		wg.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
207
}
208

209
func assertCanGet(t *testing.T, ds DAGService, n Node) {
Jeromy's avatar
Jeromy committed
210
	if _, err := ds.Get(context.Background(), n.Cid()); err != nil {
211 212 213 214 215
		t.Fatal(err)
	}
}

func TestCantGet(t *testing.T) {
216
	ds := dstest.Mock()
217
	a := NodeWithData([]byte("A"))
218

Jeromy's avatar
Jeromy committed
219 220
	c := a.Cid()
	_, err := ds.Get(context.Background(), c)
221 222 223 224
	if !strings.Contains(err.Error(), "not found") {
		t.Fatal("expected err not found, got: ", err)
	}
}
225 226

func TestFetchGraph(t *testing.T) {
Jeromy's avatar
Jeromy committed
227
	var dservs []DAGService
228
	bsis := bstest.Mocks(2)
Jeromy's avatar
Jeromy committed
229 230 231
	for _, bsi := range bsis {
		dservs = append(dservs, NewDAGService(bsi))
	}
232 233

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
Jeromy's avatar
Jeromy committed
234
	root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
235 236 237 238
	if err != nil {
		t.Fatal(err)
	}

239
	err = FetchGraph(context.TODO(), root.Cid(), dservs[1])
240 241 242 243
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
244
	// create an offline dagstore and ensure all blocks were fetched
245
	bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore()))
246

Jeromy's avatar
Jeromy committed
247 248
	offline_ds := NewDAGService(bs)

249
	err = EnumerateChildren(context.Background(), offline_ds, root.Cid(), func(_ *cid.Cid) bool { return true }, false)
250 251 252 253 254 255
	if err != nil {
		t.Fatal(err)
	}
}

func TestEnumerateChildren(t *testing.T) {
256
	bsi := bstest.Mocks(1)
257 258 259
	ds := NewDAGService(bsi[0])

	read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
Jeromy's avatar
Jeromy committed
260
	root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
261 262 263 264
	if err != nil {
		t.Fatal(err)
	}

Jeromy's avatar
Jeromy committed
265
	set := cid.NewSet()
266
	err = EnumerateChildren(context.Background(), ds, root.Cid(), set.Visit, false)
267 268 269 270
	if err != nil {
		t.Fatal(err)
	}

271 272
	var traverse func(n Node)
	traverse = func(n Node) {
273
		// traverse dag and check
274 275
		for _, lnk := range n.Links() {
			c := lnk.Cid
Jeromy's avatar
Jeromy committed
276
			if !set.Has(c) {
277
				t.Fatal("missing key in set! ", lnk.Cid.String())
278
			}
Jeromy's avatar
Jeromy committed
279
			child, err := ds.Get(context.Background(), c)
280 281 282 283 284 285 286 287 288
			if err != nil {
				t.Fatal(err)
			}
			traverse(child)
		}
	}

	traverse(root)
}
289 290 291 292 293

func TestFetchFailure(t *testing.T) {
	ds := dstest.Mock()
	ds_bad := dstest.Mock()

294
	top := new(ProtoNode)
295
	for i := 0; i < 10; i++ {
296
		nd := NodeWithData([]byte{byte('a' + i)})
297 298 299 300 301 302 303 304 305 306 307 308
		_, err := ds.Add(nd)
		if err != nil {
			t.Fatal(err)
		}

		err = top.AddNodeLinkClean(fmt.Sprintf("AA%d", i), nd)
		if err != nil {
			t.Fatal(err)
		}
	}

	for i := 0; i < 10; i++ {
309
		nd := NodeWithData([]byte{'f', 'a' + byte(i)})
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
		_, err := ds_bad.Add(nd)
		if err != nil {
			t.Fatal(err)
		}

		err = top.AddNodeLinkClean(fmt.Sprintf("BB%d", i), nd)
		if err != nil {
			t.Fatal(err)
		}
	}

	getters := GetDAG(context.Background(), ds, top)
	for i, getter := range getters {
		_, err := getter.Get(context.Background())
		if err != nil && i < 10 {
			t.Fatal(err)
		}
		if err == nil && i >= 10 {
			t.Fatal("should have failed request")
		}
	}
}
332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352

func TestUnmarshalFailure(t *testing.T) {
	badData := []byte("hello world")

	_, err := DecodeProtobuf(badData)
	if err == nil {
		t.Fatal("shouldnt succeed to parse this")
	}

	// now with a bad link
	pbn := &mdpb.PBNode{Links: []*mdpb.PBLink{{Hash: []byte("not a multihash")}}}
	badlink, err := pbn.Marshal()
	if err != nil {
		t.Fatal(err)
	}

	_, err = DecodeProtobuf(badlink)
	if err == nil {
		t.Fatal("should have failed to parse node with bad link")
	}

353
	n := &ProtoNode{}
354 355
	n.Marshal()
}
Jeromy's avatar
Jeromy committed
356 357 358

func TestBasicAddGet(t *testing.T) {
	ds := dstest.Mock()
359
	nd := new(ProtoNode)
Jeromy's avatar
Jeromy committed
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374

	c, err := ds.Add(nd)
	if err != nil {
		t.Fatal(err)
	}

	out, err := ds.Get(context.Background(), c)
	if err != nil {
		t.Fatal(err)
	}

	if !nd.Cid().Equals(out.Cid()) {
		t.Fatal("output didnt match input")
	}
}