flatfs_test.go 26.7 KB
Newer Older
Tommi Virtanen's avatar
Tommi Virtanen committed
1 2 3
package flatfs_test

import (
Jeromy's avatar
Jeromy committed
4
	"encoding/base32"
5
	"encoding/json"
6
	"fmt"
Tommi Virtanen's avatar
Tommi Virtanen committed
7
	"io/ioutil"
8
	"math"
9
	"math/rand"
Tommi Virtanen's avatar
Tommi Virtanen committed
10 11
	"os"
	"path/filepath"
12
	"runtime"
13 14
	"strings"
	"sync"
Tommi Virtanen's avatar
Tommi Virtanen committed
15
	"testing"
16
	"time"
Tommi Virtanen's avatar
Tommi Virtanen committed
17

Jeromy's avatar
Jeromy committed
18
	"github.com/ipfs/go-datastore"
19
	"github.com/ipfs/go-datastore/mount"
Jeromy's avatar
Jeromy committed
20 21
	"github.com/ipfs/go-datastore/query"
	dstest "github.com/ipfs/go-datastore/test"
22

Jakub Sztandera's avatar
Jakub Sztandera committed
23
	"github.com/ipfs/go-ds-flatfs"
Tommi Virtanen's avatar
Tommi Virtanen committed
24 25
)

26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
func checkTemp(t *testing.T, dir string) {
	tempDir, err := os.Open(filepath.Join(dir, ".temp"))
	if err != nil {
		t.Errorf("failed to open temp dir: %s", err)
		return
	}

	names, err := tempDir.Readdirnames(-1)
	tempDir.Close()

	if err != nil {
		t.Errorf("failed to read temp dir: %s", err)
		return
	}

	for _, name := range names {
		t.Errorf("found leftover temporary file: %s", name)
	}
}

Tommi Virtanen's avatar
Tommi Virtanen committed
46 47 48 49 50 51 52 53 54 55 56 57 58 59
func tempdir(t testing.TB) (path string, cleanup func()) {
	path, err := ioutil.TempDir("", "test-datastore-flatfs-")
	if err != nil {
		t.Fatalf("cannot create temp directory: %v", err)
	}

	cleanup = func() {
		if err := os.RemoveAll(path); err != nil {
			t.Errorf("tempdir cleanup failed: %v", err)
		}
	}
	return path, cleanup
}

60 61 62 63
func tryAllShardFuncs(t *testing.T, testFunc func(mkShardFunc, *testing.T)) {
	t.Run("prefix", func(t *testing.T) { testFunc(flatfs.Prefix, t) })
	t.Run("suffix", func(t *testing.T) { testFunc(flatfs.Suffix, t) })
	t.Run("next-to-last", func(t *testing.T) { testFunc(flatfs.NextToLast, t) })
64 65
}

66 67
type mkShardFunc func(int) *flatfs.ShardIdV1

68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
func testBatch(dirFunc mkShardFunc, t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
	defer checkTemp(t, temp)

	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
	defer fs.Close()

	batches := make([]datastore.Batch, 9)
	for i := range batches {
		batch, err := fs.Batch()
		if err != nil {
			t.Fatal(err)
		}

		batches[i] = batch

		err = batch.Put(datastore.NewKey("QUUX"), []byte("foo"))
		if err != nil {
			t.Fatal(err)
		}
		err = batch.Put(datastore.NewKey(fmt.Sprintf("Q%dX", i)), []byte(fmt.Sprintf("bar%d", i)))
		if err != nil {
			t.Fatal(err)
		}
	}

	var wg sync.WaitGroup
	wg.Add(len(batches))
	for _, batch := range batches {
		batch := batch
		go func() {
			defer wg.Done()
			err := batch.Commit()
			if err != nil {
				t.Error(err)
			}
		}()
	}

	check := func(key, expected string) {
		actual, err := fs.Get(datastore.NewKey(key))
		if err != nil {
			t.Fatalf("get for key %s, error: %s", key, err)
		}
		if string(actual) != expected {
			t.Fatalf("for key %s, expected %s, got %s", key, expected, string(actual))
		}
	}

	wg.Wait()

	check("QUUX", "foo")
	for i := range batches {
		check(fmt.Sprintf("Q%dX", i), fmt.Sprintf("bar%d", i))
	}
}

func TestBatch(t *testing.T) { tryAllShardFuncs(t, testBatch) }

131
func testPut(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
132 133
	temp, cleanup := tempdir(t)
	defer cleanup()
134
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
135

136
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
137 138 139
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
140
	defer fs.Close()
Tommi Virtanen's avatar
Tommi Virtanen committed
141

142
	err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
Tommi Virtanen's avatar
Tommi Virtanen committed
143 144 145
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}
146 147 148 149 150

	err = fs.Put(datastore.NewKey("foo"), []byte("nonono"))
	if err == nil {
		t.Fatalf("did not expect to put a lowercase key")
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
151 152
}

153
func TestPut(t *testing.T) { tryAllShardFuncs(t, testPut) }
154

155
func testGet(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
156 157
	temp, cleanup := tempdir(t)
	defer cleanup()
158
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
159

160
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
161 162 163
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
164
	defer fs.Close()
Tommi Virtanen's avatar
Tommi Virtanen committed
165 166

	const input = "foobar"
167
	err = fs.Put(datastore.NewKey("QUUX"), []byte(input))
Tommi Virtanen's avatar
Tommi Virtanen committed
168 169 170 171
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

172
	buf, err := fs.Get(datastore.NewKey("QUUX"))
Tommi Virtanen's avatar
Tommi Virtanen committed
173 174 175 176 177 178
	if err != nil {
		t.Fatalf("Get failed: %v", err)
	}
	if g, e := string(buf), input; g != e {
		t.Fatalf("Get gave wrong content: %q != %q", g, e)
	}
179 180 181 182 183

	_, err = fs.Get(datastore.NewKey("/FOO/BAR"))
	if err != datastore.ErrNotFound {
		t.Fatalf("expected ErrNotFound, got %s", err)
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
184 185
}

186
func TestGet(t *testing.T) { tryAllShardFuncs(t, testGet) }
187

188
func testPutOverwrite(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
189 190
	temp, cleanup := tempdir(t)
	defer cleanup()
191
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
192

193
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
194 195 196
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
197
	defer fs.Close()
Tommi Virtanen's avatar
Tommi Virtanen committed
198 199 200 201 202

	const (
		loser  = "foobar"
		winner = "xyzzy"
	)
203
	err = fs.Put(datastore.NewKey("QUUX"), []byte(loser))
Tommi Virtanen's avatar
Tommi Virtanen committed
204 205 206 207
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

208
	err = fs.Put(datastore.NewKey("QUUX"), []byte(winner))
Tommi Virtanen's avatar
Tommi Virtanen committed
209 210 211 212
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

213
	data, err := fs.Get(datastore.NewKey("QUUX"))
Tommi Virtanen's avatar
Tommi Virtanen committed
214 215 216
	if err != nil {
		t.Fatalf("Get failed: %v", err)
	}
217
	if g, e := string(data), winner; g != e {
Tommi Virtanen's avatar
Tommi Virtanen committed
218 219 220 221
		t.Fatalf("Get gave wrong content: %q != %q", g, e)
	}
}

222
func TestPutOverwrite(t *testing.T) { tryAllShardFuncs(t, testPutOverwrite) }
223

224
func testGetNotFoundError(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
225 226
	temp, cleanup := tempdir(t)
	defer cleanup()
227
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
228

229
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
230 231 232
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
233
	defer fs.Close()
Tommi Virtanen's avatar
Tommi Virtanen committed
234

235
	_, err = fs.Get(datastore.NewKey("QUUX"))
Tommi Virtanen's avatar
Tommi Virtanen committed
236 237 238 239 240
	if g, e := err, datastore.ErrNotFound; g != e {
		t.Fatalf("expected ErrNotFound, got: %v\n", g)
	}
}

241
func TestGetNotFoundError(t *testing.T) { tryAllShardFuncs(t, testGetNotFoundError) }
242 243

type params struct {
244
	shard *flatfs.ShardIdV1
Kevin Atkinson's avatar
Kevin Atkinson committed
245 246
	dir   string
	key   string
247 248 249
}

func testStorage(p *params, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
250 251
	temp, cleanup := tempdir(t)
	defer cleanup()
252
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
253

254
	target := p.dir + string(os.PathSeparator) + p.key + ".data"
255
	fs, err := flatfs.CreateOrOpen(temp, p.shard, false)
Tommi Virtanen's avatar
Tommi Virtanen committed
256 257 258
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
259
	defer fs.Close()
Tommi Virtanen's avatar
Tommi Virtanen committed
260

261
	err = fs.Put(datastore.NewKey(p.key), []byte("foobar"))
Tommi Virtanen's avatar
Tommi Virtanen committed
262 263 264 265
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

266
	fs.Close()
Tommi Virtanen's avatar
Tommi Virtanen committed
267
	seen := false
268
	haveREADME := false
Tommi Virtanen's avatar
Tommi Virtanen committed
269 270 271 272 273 274 275 276 277
	walk := func(absPath string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		path, err := filepath.Rel(temp, absPath)
		if err != nil {
			return err
		}
		switch path {
278
		case ".", "..", "SHARDING", flatfs.DiskUsageFile, ".temp":
Tommi Virtanen's avatar
Tommi Virtanen committed
279
			// ignore
Kevin Atkinson's avatar
Kevin Atkinson committed
280
		case "_README":
281 282 283 284 285
			_, err := ioutil.ReadFile(absPath)
			if err != nil {
				t.Error("could not read _README file")
			}
			haveREADME = true
286
		case p.dir:
Tommi Virtanen's avatar
Tommi Virtanen committed
287
			if !fi.IsDir() {
Kevin Atkinson's avatar
Kevin Atkinson committed
288
				t.Errorf("directory is not a file? %v", fi.Mode())
Tommi Virtanen's avatar
Tommi Virtanen committed
289 290 291 292 293 294 295 296
			}
			// we know it's there if we see the file, nothing more to
			// do here
		case target:
			seen = true
			if !fi.Mode().IsRegular() {
				t.Errorf("expected a regular file, mode: %04o", fi.Mode())
			}
297 298 299 300
			if runtime.GOOS != "windows" {
				if g, e := fi.Mode()&os.ModePerm&0007, os.FileMode(0000); g != e {
					t.Errorf("file should not be world accessible: %04o", fi.Mode())
				}
Tommi Virtanen's avatar
Tommi Virtanen committed
301 302 303 304 305 306 307
			}
		default:
			t.Errorf("saw unexpected directory entry: %q %v", path, fi.Mode())
		}
		return nil
	}
	if err := filepath.Walk(temp, walk); err != nil {
308
		t.Fatalf("walk: %v", err)
Tommi Virtanen's avatar
Tommi Virtanen committed
309 310 311 312
	}
	if !seen {
		t.Error("did not see the data file")
	}
313
	if fs.ShardStr() == flatfs.IPFS_DEF_SHARD_STR && !haveREADME {
314
		t.Error("expected _README file")
315
	} else if fs.ShardStr() != flatfs.IPFS_DEF_SHARD_STR && haveREADME {
316 317
		t.Error("did not expect _README file")
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
318
}
Tommi Virtanen's avatar
Tommi Virtanen committed
319

320 321 322
func TestStorage(t *testing.T) {
	t.Run("prefix", func(t *testing.T) {
		testStorage(&params{
323
			shard: flatfs.Prefix(2),
324 325
			dir:   "QU",
			key:   "QUUX",
326 327 328 329
		}, t)
	})
	t.Run("suffix", func(t *testing.T) {
		testStorage(&params{
330
			shard: flatfs.Suffix(2),
331 332
			dir:   "UX",
			key:   "QUUX",
333 334
		}, t)
	})
335 336
	t.Run("next-to-last", func(t *testing.T) {
		testStorage(&params{
337
			shard: flatfs.NextToLast(2),
338 339
			dir:   "UU",
			key:   "QUUX",
340 341
		}, t)
	})
342 343
}

344
func testHasNotFound(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
345 346
	temp, cleanup := tempdir(t)
	defer cleanup()
347
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
348

349
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
350 351 352
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
353
	defer fs.Close()
Tommi Virtanen's avatar
Tommi Virtanen committed
354

355
	found, err := fs.Has(datastore.NewKey("QUUX"))
Tommi Virtanen's avatar
Tommi Virtanen committed
356 357 358
	if err != nil {
		t.Fatalf("Has fail: %v\n", err)
	}
Steven Allen's avatar
Steven Allen committed
359 360
	if found {
		t.Fatal("Has should have returned false")
Tommi Virtanen's avatar
Tommi Virtanen committed
361 362 363
	}
}

364
func TestHasNotFound(t *testing.T) { tryAllShardFuncs(t, testHasNotFound) }
365

366
func testHasFound(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
367 368
	temp, cleanup := tempdir(t)
	defer cleanup()
369
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
370

371
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
372 373 374
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
375 376
	defer fs.Close()

377
	err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
Tommi Virtanen's avatar
Tommi Virtanen committed
378 379 380 381
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

382
	found, err := fs.Has(datastore.NewKey("QUUX"))
Tommi Virtanen's avatar
Tommi Virtanen committed
383 384 385
	if err != nil {
		t.Fatalf("Has fail: %v\n", err)
	}
Steven Allen's avatar
Steven Allen committed
386 387
	if !found {
		t.Fatal("Has should have returned true")
Tommi Virtanen's avatar
Tommi Virtanen committed
388 389
	}
}
Tommi Virtanen's avatar
Tommi Virtanen committed
390

391
func TestHasFound(t *testing.T) { tryAllShardFuncs(t, testHasFound) }
392

Steven Allen's avatar
Steven Allen committed
393 394 395
func testGetSizeFound(dirFunc mkShardFunc, t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
396
	defer checkTemp(t, temp)
Steven Allen's avatar
Steven Allen committed
397 398 399 400 401 402 403

	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
	defer fs.Close()

404
	_, err = fs.GetSize(datastore.NewKey("QUUX"))
Steven Allen's avatar
Steven Allen committed
405 406 407 408 409 410 411 412 413 414
	if err != datastore.ErrNotFound {
		t.Fatalf("GetSize should have returned ErrNotFound, got: %v\n", err)
	}
}

func TestGetSizeFound(t *testing.T) { tryAllShardFuncs(t, testGetSizeFound) }

func testGetSizeNotFound(dirFunc mkShardFunc, t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
415
	defer checkTemp(t, temp)
Steven Allen's avatar
Steven Allen committed
416 417 418 419 420 421 422

	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
	defer fs.Close()

423
	err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
Steven Allen's avatar
Steven Allen committed
424 425 426 427
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

428
	size, err := fs.GetSize(datastore.NewKey("QUUX"))
Steven Allen's avatar
Steven Allen committed
429 430 431 432 433 434 435 436 437 438
	if err != nil {
		t.Fatalf("GetSize failed with: %v\n", err)
	}
	if size != len("foobar") {
		t.Fatalf("GetSize returned wrong size: got %d, expected %d", size, len("foobar"))
	}
}

func TestGetSizeNotFound(t *testing.T) { tryAllShardFuncs(t, testGetSizeNotFound) }

439
func testDeleteNotFound(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
440 441
	temp, cleanup := tempdir(t)
	defer cleanup()
442
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
443

444
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
445 446 447
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
448
	defer fs.Close()
Tommi Virtanen's avatar
Tommi Virtanen committed
449

450
	err = fs.Delete(datastore.NewKey("QUUX"))
Steven Allen's avatar
Steven Allen committed
451 452
	if err != nil {
		t.Fatalf("expected nil, got: %v\n", err)
Tommi Virtanen's avatar
Tommi Virtanen committed
453 454 455
	}
}

456
func TestDeleteNotFound(t *testing.T) { tryAllShardFuncs(t, testDeleteNotFound) }
457

458
func testDeleteFound(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
459 460
	temp, cleanup := tempdir(t)
	defer cleanup()
461
	defer checkTemp(t, temp)
Tommi Virtanen's avatar
Tommi Virtanen committed
462

463
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
464 465 466
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
467 468
	defer fs.Close()

469
	err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
Tommi Virtanen's avatar
Tommi Virtanen committed
470 471 472 473
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

474
	err = fs.Delete(datastore.NewKey("QUUX"))
Tommi Virtanen's avatar
Tommi Virtanen committed
475 476 477 478 479
	if err != nil {
		t.Fatalf("Delete fail: %v\n", err)
	}

	// check that it's gone
480
	_, err = fs.Get(datastore.NewKey("QUUX"))
Tommi Virtanen's avatar
Tommi Virtanen committed
481 482 483 484
	if g, e := err, datastore.ErrNotFound; g != e {
		t.Fatalf("expected Get after Delete to give ErrNotFound, got: %v\n", g)
	}
}
485

486
func TestDeleteFound(t *testing.T) { tryAllShardFuncs(t, testDeleteFound) }
487

488
func testQuerySimple(dirFunc mkShardFunc, t *testing.T) {
489 490
	temp, cleanup := tempdir(t)
	defer cleanup()
491
	defer checkTemp(t, temp)
492

493
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
494 495 496
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
497 498
	defer fs.Close()

499
	const myKey = "QUUX"
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
	err = fs.Put(datastore.NewKey(myKey), []byte("foobar"))
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	res, err := fs.Query(query.Query{KeysOnly: true})
	if err != nil {
		t.Fatalf("Query fail: %v\n", err)
	}
	entries, err := res.Rest()
	if err != nil {
		t.Fatalf("Query Results.Rest fail: %v\n", err)
	}
	seen := false
	for _, e := range entries {
		switch e.Key {
		case datastore.NewKey(myKey).String():
			seen = true
		default:
			t.Errorf("saw unexpected key: %q", e.Key)
		}
	}
	if !seen {
		t.Errorf("did not see wanted key %q in %+v", myKey, entries)
	}
}
Jeromy's avatar
Jeromy committed
526

527
func TestQuerySimple(t *testing.T) { tryAllShardFuncs(t, testQuerySimple) }
528

529 530 531
func testDiskUsage(dirFunc mkShardFunc, t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
532
	defer checkTemp(t, temp)
533 534 535 536 537

	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
538
	defer fs.Close()
539 540 541 542 543 544 545 546 547 548

	time.Sleep(100 * time.Millisecond)
	duNew, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}
	t.Log("duNew:", duNew)

	count := 200
	for i := 0; i < count; i++ {
549
		k := datastore.NewKey(fmt.Sprintf("TEST-%d", i))
550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
		v := []byte("10bytes---")
		err = fs.Put(k, v)
		if err != nil {
			t.Fatalf("Put fail: %v\n", err)
		}
	}

	time.Sleep(100 * time.Millisecond)
	duElems, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}
	t.Log("duPostPut:", duElems)

	for i := 0; i < count; i++ {
565
		k := datastore.NewKey(fmt.Sprintf("TEST-%d", i))
566 567 568 569 570 571 572 573 574 575 576 577 578
		err = fs.Delete(k)
		if err != nil {
			t.Fatalf("Delete fail: %v\n", err)
		}
	}

	time.Sleep(100 * time.Millisecond)
	duDelete, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}
	t.Log("duPostDelete:", duDelete)

579 580 581 582
	du, err := fs.DiskUsage()
	t.Log("duFinal:", du)
	if err != nil {
		t.Fatal(err)
583
	}
584
	fs.Close()
585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607

	// Check that disk usage file is correct
	duB, err := ioutil.ReadFile(filepath.Join(temp, flatfs.DiskUsageFile))
	if err != nil {
		t.Fatal(err)
	}
	contents := make(map[string]interface{})
	err = json.Unmarshal(duB, &contents)
	if err != nil {
		t.Fatal(err)
	}

	// Make sure diskUsage value is correct
	if val, ok := contents["diskUsage"].(float64); !ok || uint64(val) != du {
		t.Fatalf("Unexpected value for diskUsage in %s: %v (expected %d)",
			flatfs.DiskUsageFile, contents["diskUsage"], du)
	}

	// Make sure the accuracy value is correct
	if val, ok := contents["accuracy"].(string); !ok || val != "initial-exact" {
		t.Fatalf("Unexpected value for accuracyin %s: %v",
			flatfs.DiskUsageFile, contents["accuracy"])
	}
608 609

	// Make sure size is correctly calculated on re-open
610
	os.Remove(filepath.Join(temp, flatfs.DiskUsageFile))
611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653
	fs, err = flatfs.Open(temp, false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	duReopen, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}
	t.Log("duReopen:", duReopen)

	// Checks
	if duNew == 0 {
		t.Error("new datastores should have some size")
	}

	if duElems <= duNew {
		t.Error("size should grow as new elements are added")
	}

	if duElems-duDelete != uint64(count*10) {
		t.Error("size should be reduced exactly as size of objects deleted")
	}

	if duReopen < duNew {
		t.Error("Reopened datastore should not be smaller")
	}
}

func TestDiskUsage(t *testing.T) {
	tryAllShardFuncs(t, testDiskUsage)
}

func TestDiskUsageDoubleCount(t *testing.T) {
	tryAllShardFuncs(t, testDiskUsageDoubleCount)
}

// test that concurrently writing and deleting the same key/value
// does not throw any errors and disk usage does not do
// any double-counting.
func testDiskUsageDoubleCount(dirFunc mkShardFunc, t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
654
	defer checkTemp(t, temp)
655 656 657 658 659

	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
660
	defer fs.Close()
661 662 663

	var count int
	var wg sync.WaitGroup
664
	testKey := datastore.NewKey("TEST")
665 666 667 668 669 670 671

	put := func() {
		defer wg.Done()
		for i := 0; i < count; i++ {
			v := []byte("10bytes---")
			err := fs.Put(testKey, v)
			if err != nil {
Steven Allen's avatar
Steven Allen committed
672
				t.Errorf("Put fail: %v\n", err)
673 674 675 676 677 678 679 680 681
			}
		}
	}

	del := func() {
		defer wg.Done()
		for i := 0; i < count; i++ {
			err := fs.Delete(testKey)
			if err != nil && !strings.Contains(err.Error(), "key not found") {
Steven Allen's avatar
Steven Allen committed
682
				t.Errorf("Delete fail: %v\n", err)
683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
			}
		}
	}

	// Add one element and then remove it and check disk usage
	// makes sense
	count = 1
	wg.Add(2)
	put()
	du, _ := fs.DiskUsage()
	del()
	du2, _ := fs.DiskUsage()
	if du-10 != du2 {
		t.Error("should have deleted exactly 10 bytes:", du, du2)
	}

	// Add and remove many times at the same time
	count = 200
	wg.Add(4)
	go put()
	go del()
	go put()
	go del()
	wg.Wait()

	du3, _ := fs.DiskUsage()
	has, err := fs.Has(testKey)
	if err != nil {
		t.Fatal(err)
	}

	if has { // put came last
		if du3 != du {
			t.Error("du should be the same as after first put:", du, du3)
		}
	} else { //delete came last
		if du3 != du2 {
			t.Error("du should be the same as after first delete:", du2, du3)
		}
	}
}

func testDiskUsageBatch(dirFunc mkShardFunc, t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
728
	defer checkTemp(t, temp)
729 730 731 732 733

	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
734
	defer fs.Close()
735

Steven Allen's avatar
Steven Allen committed
736 737 738 739
	fsBatch, err := fs.Batch()
	if err != nil {
		t.Fatal(err)
	}
740 741 742 743 744

	count := 200
	var wg sync.WaitGroup
	testKeys := []datastore.Key{}
	for i := 0; i < count; i++ {
745
		k := datastore.NewKey(fmt.Sprintf("TEST%d", i))
746 747 748 749 750
		testKeys = append(testKeys, k)
	}

	put := func() {
		for i := 0; i < count; i++ {
Steven Allen's avatar
Steven Allen committed
751 752 753 754
			err := fsBatch.Put(testKeys[i], []byte("10bytes---"))
			if err != nil {
				t.Error(err)
			}
755 756 757 758 759 760
		}
	}
	commit := func() {
		defer wg.Done()
		err := fsBatch.Commit()
		if err != nil {
Steven Allen's avatar
Steven Allen committed
761
			t.Errorf("Batch Put fail: %v\n", err)
762 763 764 765 766 767 768 769
		}
	}

	del := func() {
		defer wg.Done()
		for _, k := range testKeys {
			err := fs.Delete(k)
			if err != nil && !strings.Contains(err.Error(), "key not found") {
Steven Allen's avatar
Steven Allen committed
770
				t.Errorf("Delete fail: %v\n", err)
771 772 773 774 775 776 777 778 779
			}
		}
	}

	// Put many elements and then delete them and check disk usage
	// makes sense
	wg.Add(2)
	put()
	commit()
Steven Allen's avatar
Steven Allen committed
780 781 782 783
	du, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}
784
	del()
Steven Allen's avatar
Steven Allen committed
785 786 787 788
	du2, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}
789 790 791 792 793 794 795 796 797 798 799
	if du-uint64(10*count) != du2 {
		t.Errorf("should have deleted exactly %d bytes: %d %d", 10*count, du, du2)
	}

	// Do deletes while doing putManys concurrently
	wg.Add(2)
	put()
	go commit()
	go del()
	wg.Wait()

Steven Allen's avatar
Steven Allen committed
800 801 802 803
	du3, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}
804 805 806 807
	// Now query how many keys we have
	results, err := fs.Query(query.Query{
		KeysOnly: true,
	})
Steven Allen's avatar
Steven Allen committed
808 809 810
	if err != nil {
		t.Fatal(err)
	}
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828
	rest, err := results.Rest()
	if err != nil {
		t.Fatal(err)
	}

	expectedSize := uint64(len(rest) * 10)

	if exp := du2 + expectedSize; exp != du3 {
		t.Error("diskUsage has skewed off from real size:",
			exp, du3)
	}
}

func TestDiskUsageBatch(t *testing.T) { tryAllShardFuncs(t, testDiskUsageBatch) }

func testDiskUsageEstimation(dirFunc mkShardFunc, t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
829
	defer checkTemp(t, temp)
830 831 832 833 834

	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
835
	defer fs.Close()
836 837 838

	count := 50000
	for i := 0; i < count; i++ {
839
		k := datastore.NewKey(fmt.Sprintf("%d-TEST-%d", i, i))
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
		v := make([]byte, 1000)
		err = fs.Put(k, v)
		if err != nil {
			t.Fatalf("Put fail: %v\n", err)
		}
	}

	// Delete checkpoint
	fs.Close()
	os.Remove(filepath.Join(temp, flatfs.DiskUsageFile))

	// This will do a full du
	flatfs.DiskUsageFilesAverage = -1
	fs, err = flatfs.Open(temp, false)
	if err != nil {
		t.Fatalf("Open fail: %v\n", err)
	}

	duReopen, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}

	fs.Close()
	os.Remove(filepath.Join(temp, flatfs.DiskUsageFile))

	// This will estimate the size. Since all files are the same
	// length we can use a low file average number.
	flatfs.DiskUsageFilesAverage = 100
	// Make sure size is correctly calculated on re-open
	fs, err = flatfs.Open(temp, false)
	if err != nil {
		t.Fatalf("Open fail: %v\n", err)
	}

	duEst, err := fs.DiskUsage()
	if err != nil {
		t.Fatal(err)
	}

	t.Log("RealDu:", duReopen)
	t.Log("Est:", duEst)

	diff := int(math.Abs(float64(int(duReopen) - int(duEst))))
	maxDiff := int(0.05 * float64(duReopen)) // %5 of actual

	if diff > maxDiff {
887
		t.Fatalf("expected a better estimation within 5%%")
888
	}
889 890 891 892 893

	// Make sure the accuracy value is correct
	if fs.Accuracy() != "initial-approximate" {
		t.Errorf("Unexpected value for fs.Accuracy(): %s", fs.Accuracy())
	}
894 895 896 897 898 899 900 901 902 903 904 905 906

	fs.Close()

	// Reopen into a new variable
	fs2, err := flatfs.Open(temp, false)
	if err != nil {
		t.Fatalf("Open fail: %v\n", err)
	}

	// Make sure the accuracy value is preserved
	if fs2.Accuracy() != "initial-approximate" {
		t.Errorf("Unexpected value for fs.Accuracy(): %s", fs2.Accuracy())
	}
907 908 909 910
}

func TestDiskUsageEstimation(t *testing.T) { tryAllShardFuncs(t, testDiskUsageEstimation) }

911
func testBatchPut(dirFunc mkShardFunc, t *testing.T) {
Jeromy's avatar
Jeromy committed
912 913
	temp, cleanup := tempdir(t)
	defer cleanup()
914
	defer checkTemp(t, temp)
Jeromy's avatar
Jeromy committed
915

916
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Jeromy's avatar
Jeromy committed
917 918 919
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
920
	defer fs.Close()
Jeromy's avatar
Jeromy committed
921

Jeromy's avatar
Jeromy committed
922 923
	dstest.RunBatchTest(t, fs)
}
Jeromy's avatar
Jeromy committed
924

925
func TestBatchPut(t *testing.T) { tryAllShardFuncs(t, testBatchPut) }
926

927
func testBatchDelete(dirFunc mkShardFunc, t *testing.T) {
Jeromy's avatar
Jeromy committed
928 929
	temp, cleanup := tempdir(t)
	defer cleanup()
930
	defer checkTemp(t, temp)
Jeromy's avatar
Jeromy committed
931

932
	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
Jeromy's avatar
Jeromy committed
933
	if err != nil {
Jeromy's avatar
Jeromy committed
934
		t.Fatalf("New fail: %v\n", err)
Jeromy's avatar
Jeromy committed
935
	}
936
	defer fs.Close()
Jeromy's avatar
Jeromy committed
937

Jeromy's avatar
Jeromy committed
938
	dstest.RunBatchDeleteTest(t, fs)
Jeromy's avatar
Jeromy committed
939 940
}

941
func TestBatchDelete(t *testing.T) { tryAllShardFuncs(t, testBatchDelete) }
942

Steven Allen's avatar
Steven Allen committed
943 944 945
func testClose(dirFunc mkShardFunc, t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
946
	defer checkTemp(t, temp)
Steven Allen's avatar
Steven Allen committed
947 948 949 950 951 952

	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

953
	err = fs.Put(datastore.NewKey("QUUX"), []byte("foobar"))
Steven Allen's avatar
Steven Allen committed
954 955 956 957 958 959
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	fs.Close()

960
	err = fs.Put(datastore.NewKey("QAAX"), []byte("foobar"))
Steven Allen's avatar
Steven Allen committed
961 962 963 964 965 966 967
	if err == nil {
		t.Fatal("expected put on closed datastore to fail")
	}
}

func TestClose(t *testing.T) { tryAllShardFuncs(t, testClose) }

968 969 970 971
func TestSHARDINGFile(t *testing.T) {
	tempdir, cleanup := tempdir(t)
	defer cleanup()

972
	fun := flatfs.IPFS_DEF_SHARD
973

974
	err := flatfs.Create(tempdir, fun)
975
	if err != nil {
976
		t.Fatalf("Create: %v\n", err)
977 978
	}

979
	fs, err := flatfs.Open(tempdir, false)
980
	if err != nil {
981
		t.Fatalf("Open fail: %v\n", err)
982
	}
983 984
	if fs.ShardStr() != flatfs.IPFS_DEF_SHARD_STR {
		t.Fatalf("Expected '%s' for shard function got '%s'", flatfs.IPFS_DEF_SHARD_STR, fs.ShardStr())
985 986 987
	}
	fs.Close()

988
	fs, err = flatfs.CreateOrOpen(tempdir, fun, false)
989 990 991 992 993
	if err != nil {
		t.Fatalf("Could not reopen repo: %v\n", err)
	}
	fs.Close()

994
	fs, err = flatfs.CreateOrOpen(tempdir, flatfs.Prefix(5), false)
995
	if err == nil {
Steven Allen's avatar
Steven Allen committed
996
		fs.Close()
997 998 999 1000
		t.Fatalf("Was able to open repo with incompatible sharding function")
	}
}

1001
func TestInvalidPrefix(t *testing.T) {
1002
	_, err := flatfs.ParseShardFunc("/bad/prefix/v1/next-to-last/2")
1003
	if err == nil {
1004
		t.Fatalf("Expected an error while parsing a shard identifier with a bad prefix")
1005 1006 1007
	}
}

1008 1009 1010 1011
func TestNonDatastoreDir(t *testing.T) {
	tempdir, cleanup := tempdir(t)
	defer cleanup()

Steven Allen's avatar
Steven Allen committed
1012 1013 1014 1015
	err := ioutil.WriteFile(filepath.Join(tempdir, "afile"), []byte("Some Content"), 0644)
	if err != nil {
		t.Fatal(err)
	}
1016

Steven Allen's avatar
Steven Allen committed
1017
	err = flatfs.Create(tempdir, flatfs.NextToLast(2))
1018 1019 1020 1021 1022
	if err == nil {
		t.Fatalf("Expected an error when creating a datastore in a non-empty directory")
	}
}

1023 1024 1025
func TestNoCluster(t *testing.T) {
	tempdir, cleanup := tempdir(t)
	defer cleanup()
1026
	defer checkTemp(t, tempdir)
1027

1028
	fs, err := flatfs.CreateOrOpen(tempdir, flatfs.NextToLast(1), false)
1029 1030 1031
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
1032
	defer fs.Close()
1033

1034
	r := rand.New(rand.NewSource(0))
1035
	N := 3200 // should be divisible by 32 so the math works out
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
	for i := 0; i < N; i++ {
		blk := make([]byte, 1000)
		r.Read(blk)

		key := "CIQ" + base32.StdEncoding.EncodeToString(blk[:10])
		err := fs.Put(datastore.NewKey(key), blk)
		if err != nil {
			t.Fatalf("Put fail: %v\n", err)
		}
	}

Kevin Atkinson's avatar
Kevin Atkinson committed
1047
	fs.Close()
1048 1049 1050 1051 1052
	dirs, err := ioutil.ReadDir(tempdir)
	if err != nil {
		t.Fatalf("ReadDir fail: %v\n", err)
	}
	idealFilesPerDir := float64(N) / 32.0
1053
	tolerance := math.Floor(idealFilesPerDir * 0.25)
1054
	count := 0
1055
	for _, dir := range dirs {
1056 1057
		switch dir.Name() {
		case flatfs.SHARDING_FN, flatfs.README_FN, flatfs.DiskUsageFile, ".temp":
1058 1059 1060
			continue
		}
		count += 1
1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
		files, err := ioutil.ReadDir(filepath.Join(tempdir, dir.Name()))
		if err != nil {
			t.Fatalf("ReadDir fail: %v\n", err)
		}
		num := float64(len(files))
		if math.Abs(num-idealFilesPerDir) > tolerance {
			t.Fatalf("Dir %s has %.0f files, expected between %.f and %.f files",
				filepath.Join(tempdir, dir.Name()), num, idealFilesPerDir-tolerance, idealFilesPerDir+tolerance)
		}
	}
1071 1072 1073
	if count != 32 {
		t.Fatalf("Expected 32 directories and one file in %s", tempdir)
	}
1074 1075
}

Jeromy's avatar
Jeromy committed
1076
func BenchmarkConsecutivePut(b *testing.B) {
1077
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
Jeromy's avatar
Jeromy committed
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
	var blocks [][]byte
	var keys []datastore.Key
	for i := 0; i < b.N; i++ {
		blk := make([]byte, 256*1024)
		r.Read(blk)
		blocks = append(blocks, blk)

		key := base32.StdEncoding.EncodeToString(blk[:8])
		keys = append(keys, datastore.NewKey(key))
	}
	temp, cleanup := tempdir(b)
	defer cleanup()

1091
	fs, err := flatfs.CreateOrOpen(temp, flatfs.Prefix(2), false)
Jeromy's avatar
Jeromy committed
1092 1093 1094
	if err != nil {
		b.Fatalf("New fail: %v\n", err)
	}
1095
	defer fs.Close()
Jeromy's avatar
Jeromy committed
1096 1097 1098 1099 1100 1101 1102 1103 1104

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		err := fs.Put(keys[i], blocks[i])
		if err != nil {
			b.Fatal(err)
		}
	}
1105
	b.StopTimer() // avoid counting cleanup
Jeromy's avatar
Jeromy committed
1106 1107 1108
}

func BenchmarkBatchedPut(b *testing.B) {
1109
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
Jeromy's avatar
Jeromy committed
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
	var blocks [][]byte
	var keys []datastore.Key
	for i := 0; i < b.N; i++ {
		blk := make([]byte, 256*1024)
		r.Read(blk)
		blocks = append(blocks, blk)

		key := base32.StdEncoding.EncodeToString(blk[:8])
		keys = append(keys, datastore.NewKey(key))
	}
	temp, cleanup := tempdir(b)
	defer cleanup()

1123
	fs, err := flatfs.CreateOrOpen(temp, flatfs.Prefix(2), false)
Jeromy's avatar
Jeromy committed
1124 1125 1126
	if err != nil {
		b.Fatalf("New fail: %v\n", err)
	}
1127
	defer fs.Close()
Jeromy's avatar
Jeromy committed
1128 1129 1130 1131

	b.ResetTimer()

	for i := 0; i < b.N; {
Jeromy's avatar
Jeromy committed
1132 1133 1134 1135
		batch, err := fs.Batch()
		if err != nil {
			b.Fatal(err)
		}
Jeromy's avatar
Jeromy committed
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147

		for n := i; i-n < 512 && i < b.N; i++ {
			err := batch.Put(keys[i], blocks[i])
			if err != nil {
				b.Fatal(err)
			}
		}
		err = batch.Commit()
		if err != nil {
			b.Fatal(err)
		}
	}
1148
	b.StopTimer() // avoid counting cleanup
Jeromy's avatar
Jeromy committed
1149
}
Steven Allen's avatar
Steven Allen committed
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180

func TestQueryLeak(t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()

	fs, err := flatfs.CreateOrOpen(temp, flatfs.Prefix(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
	defer fs.Close()

	for i := 0; i < 1000; i++ {
		err = fs.Put(datastore.NewKey(fmt.Sprint(i)), []byte("foobar"))
		if err != nil {
			t.Fatalf("Put fail: %v\n", err)
		}
	}

	before := runtime.NumGoroutine()
	for i := 0; i < 200; i++ {
		res, err := fs.Query(query.Query{KeysOnly: true})
		if err != nil {
			t.Errorf("Query fail: %v\n", err)
		}
		res.Close()
	}
	after := runtime.NumGoroutine()
	if after-before > 100 {
		t.Errorf("leaked %d goroutines", after-before)
	}
}
1181 1182 1183 1184

func TestSuite(t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()
1185
	defer checkTemp(t, temp)
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207

	fs, err := flatfs.CreateOrOpen(temp, flatfs.Prefix(2), false)
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	ds := mount.New([]mount.Mount{{
		Prefix:    datastore.RawKey("/"),
		Datastore: datastore.NewMapDatastore(),
	}, {
		Prefix:    datastore.RawKey("/capital"),
		Datastore: fs,
	}})
	defer func() {
		err := ds.Close()
		if err != nil {
			t.Error(err)
		}
	}()

	dstest.SubtestAll(t, ds)
}