flatfs_test.go 10.5 KB
Newer Older
Tommi Virtanen's avatar
Tommi Virtanen committed
1 2 3
package flatfs_test

import (
Jeromy's avatar
Jeromy committed
4
	"encoding/base32"
Tommi Virtanen's avatar
Tommi Virtanen committed
5 6 7
	"io/ioutil"
	"os"
	"path/filepath"
8
	"runtime"
Tommi Virtanen's avatar
Tommi Virtanen committed
9 10
	"testing"

Jeromy's avatar
Jeromy committed
11 12 13
	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/query"
	dstest "github.com/ipfs/go-datastore/test"
Jakub Sztandera's avatar
Jakub Sztandera committed
14
	"github.com/ipfs/go-ds-flatfs"
Jeromy's avatar
Jeromy committed
15

Jakub Sztandera's avatar
Jakub Sztandera committed
16
	rand "github.com/dustin/randbo"
Tommi Virtanen's avatar
Tommi Virtanen committed
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
)

func tempdir(t testing.TB) (path string, cleanup func()) {
	path, err := ioutil.TempDir("", "test-datastore-flatfs-")
	if err != nil {
		t.Fatalf("cannot create temp directory: %v", err)
	}

	cleanup = func() {
		if err := os.RemoveAll(path); err != nil {
			t.Errorf("tempdir cleanup failed: %v", err)
		}
	}
	return path, cleanup
}

33 34 35
func tryAllShardFuncs(t *testing.T, testFunc func(mkShardFunc, *testing.T)) {
	t.Run("prefix", func(t *testing.T) { testFunc(flatfs.Prefix, t) })
	t.Run("suffix", func(t *testing.T) { testFunc(flatfs.Suffix, t) })
36
	t.Run("next-to-last", func(t *testing.T) { testFunc(flatfs.NextToLast, t) })
37 38
}

Tommi Virtanen's avatar
Tommi Virtanen committed
39 40 41 42
func TestPutBadValueType(t *testing.T) {
	temp, cleanup := tempdir(t)
	defer cleanup()

43
	fs, err := flatfs.New(temp, flatfs.Prefix(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
44 45 46 47 48 49 50 51 52 53
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	err = fs.Put(datastore.NewKey("quux"), 22)
	if g, e := err, datastore.ErrInvalidType; g != e {
		t.Fatalf("expected ErrInvalidType, got: %v\n", g)
	}
}

54 55 56
type mkShardFunc func(int) flatfs.ShardFunc

func testPut(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
57 58 59
	temp, cleanup := tempdir(t)
	defer cleanup()

60
	fs, err := flatfs.New(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
61 62 63 64 65 66 67 68 69 70
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}
}

71
func TestPut(t *testing.T) { tryAllShardFuncs(t, testPut) }
72 73

func testGet(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
74 75 76
	temp, cleanup := tempdir(t)
	defer cleanup()

77
	fs, err := flatfs.New(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	const input = "foobar"
	err = fs.Put(datastore.NewKey("quux"), []byte(input))
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	data, err := fs.Get(datastore.NewKey("quux"))
	if err != nil {
		t.Fatalf("Get failed: %v", err)
	}
	buf, ok := data.([]byte)
	if !ok {
		t.Fatalf("expected []byte from Get, got %T: %v", data, data)
	}
	if g, e := string(buf), input; g != e {
		t.Fatalf("Get gave wrong content: %q != %q", g, e)
	}
}

101
func TestGet(t *testing.T) { tryAllShardFuncs(t, testGet) }
102 103

func testPutOverwrite(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
104 105 106
	temp, cleanup := tempdir(t)
	defer cleanup()

107
	fs, err := flatfs.New(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	const (
		loser  = "foobar"
		winner = "xyzzy"
	)
	err = fs.Put(datastore.NewKey("quux"), []byte(loser))
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	err = fs.Put(datastore.NewKey("quux"), []byte(winner))
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	data, err := fs.Get(datastore.NewKey("quux"))
	if err != nil {
		t.Fatalf("Get failed: %v", err)
	}
	if g, e := string(data.([]byte)), winner; g != e {
		t.Fatalf("Get gave wrong content: %q != %q", g, e)
	}
}

135
func TestPutOverwrite(t *testing.T) { tryAllShardFuncs(t, testPutOverwrite) }
136 137

func testGetNotFoundError(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
138 139 140
	temp, cleanup := tempdir(t)
	defer cleanup()

141
	fs, err := flatfs.New(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
142 143 144 145 146 147 148 149 150 151
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	_, err = fs.Get(datastore.NewKey("quux"))
	if g, e := err, datastore.ErrNotFound; g != e {
		t.Fatalf("expected ErrNotFound, got: %v\n", g)
	}
}

152
func TestGetNotFoundError(t *testing.T) { tryAllShardFuncs(t, testGetNotFoundError) }
153 154 155 156 157 158 159 160 161

type params struct {
	what    string
	dir     string
	key     string
	dirFunc mkShardFunc
}

func testStorage(p *params, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
162 163 164
	temp, cleanup := tempdir(t)
	defer cleanup()

165 166
	target := p.dir + string(os.PathSeparator) + p.key + ".data"
	fs, err := flatfs.New(temp, p.dirFunc(len(p.dir)), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
167 168 169 170
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

171
	err = fs.Put(datastore.NewKey(p.key), []byte("foobar"))
Tommi Virtanen's avatar
Tommi Virtanen committed
172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	seen := false
	walk := func(absPath string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		path, err := filepath.Rel(temp, absPath)
		if err != nil {
			return err
		}
		switch path {
		case ".", "..":
			// ignore
188
		case p.dir:
Tommi Virtanen's avatar
Tommi Virtanen committed
189
			if !fi.IsDir() {
190
				t.Errorf("%s directory is not a file? %v", p.what, fi.Mode())
Tommi Virtanen's avatar
Tommi Virtanen committed
191 192 193 194 195 196 197 198
			}
			// we know it's there if we see the file, nothing more to
			// do here
		case target:
			seen = true
			if !fi.Mode().IsRegular() {
				t.Errorf("expected a regular file, mode: %04o", fi.Mode())
			}
199 200 201 202
			if runtime.GOOS != "windows" {
				if g, e := fi.Mode()&os.ModePerm&0007, os.FileMode(0000); g != e {
					t.Errorf("file should not be world accessible: %04o", fi.Mode())
				}
Tommi Virtanen's avatar
Tommi Virtanen committed
203 204 205 206 207 208 209 210 211 212 213 214 215
			}
		default:
			t.Errorf("saw unexpected directory entry: %q %v", path, fi.Mode())
		}
		return nil
	}
	if err := filepath.Walk(temp, walk); err != nil {
		t.Fatal("walk: %v", err)
	}
	if !seen {
		t.Error("did not see the data file")
	}
}
Tommi Virtanen's avatar
Tommi Virtanen committed
216

217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
func TestStorage(t *testing.T) {
	t.Run("prefix", func(t *testing.T) {
		testStorage(&params{
			what:    "prefix",
			dir:     "qu",
			key:     "quux",
			dirFunc: flatfs.Prefix,
		}, t)
	})
	t.Run("suffix", func(t *testing.T) {
		testStorage(&params{
			what:    "suffix",
			dir:     "ux",
			key:     "quux",
			dirFunc: flatfs.Suffix,
		}, t)
	})
234 235 236 237 238 239 240 241
	t.Run("next-to-last", func(t *testing.T) {
		testStorage(&params{
			what:    "next-to-last",
			dir:     "uu",
			key:     "quux",
			dirFunc: flatfs.NextToLast,
		}, t)
	})
242 243 244
}

func testHasNotFound(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
245 246 247
	temp, cleanup := tempdir(t)
	defer cleanup()

248
	fs, err := flatfs.New(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
249 250 251 252 253 254 255 256 257 258 259 260 261
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	found, err := fs.Has(datastore.NewKey("quux"))
	if err != nil {
		t.Fatalf("Has fail: %v\n", err)
	}
	if g, e := found, false; g != e {
		t.Fatalf("wrong Has: %v != %v", g, e)
	}
}

262
func TestHasNotFound(t *testing.T) { tryAllShardFuncs(t, testHasNotFound) }
263 264

func testHasFound(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
265 266 267
	temp, cleanup := tempdir(t)
	defer cleanup()

268
	fs, err := flatfs.New(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
	err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	found, err := fs.Has(datastore.NewKey("quux"))
	if err != nil {
		t.Fatalf("Has fail: %v\n", err)
	}
	if g, e := found, true; g != e {
		t.Fatalf("wrong Has: %v != %v", g, e)
	}
}
Tommi Virtanen's avatar
Tommi Virtanen committed
285

286
func TestHasFound(t *testing.T) { tryAllShardFuncs(t, testHasFound) }
287 288

func testDeleteNotFound(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
289 290 291
	temp, cleanup := tempdir(t)
	defer cleanup()

292
	fs, err := flatfs.New(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
293 294 295 296 297 298 299 300 301 302
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

	err = fs.Delete(datastore.NewKey("quux"))
	if g, e := err, datastore.ErrNotFound; g != e {
		t.Fatalf("expected ErrNotFound, got: %v\n", g)
	}
}

303
func TestDeleteNotFound(t *testing.T) { tryAllShardFuncs(t, testDeleteNotFound) }
304 305

func testDeleteFound(dirFunc mkShardFunc, t *testing.T) {
Tommi Virtanen's avatar
Tommi Virtanen committed
306 307 308
	temp, cleanup := tempdir(t)
	defer cleanup()

309
	fs, err := flatfs.New(temp, dirFunc(2), false)
Tommi Virtanen's avatar
Tommi Virtanen committed
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
	err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	err = fs.Delete(datastore.NewKey("quux"))
	if err != nil {
		t.Fatalf("Delete fail: %v\n", err)
	}

	// check that it's gone
	_, err = fs.Get(datastore.NewKey("quux"))
	if g, e := err, datastore.ErrNotFound; g != e {
		t.Fatalf("expected Get after Delete to give ErrNotFound, got: %v\n", g)
	}
}
329

330
func TestDeleteFound(t *testing.T) { tryAllShardFuncs(t, testDeleteFound) }
331 332

func testQuerySimple(dirFunc mkShardFunc, t *testing.T) {
333 334 335
	temp, cleanup := tempdir(t)
	defer cleanup()

336 337 338 339 340
	err := ioutil.WriteFile(filepath.Join(temp, "README"), []byte("something"), 0666)
	if err != nil {
		t.Fatalf("WriteFile fail: %v\n", err)
	}

341
	fs, err := flatfs.New(temp, dirFunc(2), false)
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}
	const myKey = "quux"
	err = fs.Put(datastore.NewKey(myKey), []byte("foobar"))
	if err != nil {
		t.Fatalf("Put fail: %v\n", err)
	}

	res, err := fs.Query(query.Query{KeysOnly: true})
	if err != nil {
		t.Fatalf("Query fail: %v\n", err)
	}
	entries, err := res.Rest()
	if err != nil {
		t.Fatalf("Query Results.Rest fail: %v\n", err)
	}
	seen := false
	for _, e := range entries {
		switch e.Key {
		case datastore.NewKey(myKey).String():
			seen = true
		default:
			t.Errorf("saw unexpected key: %q", e.Key)
		}
	}
	if !seen {
		t.Errorf("did not see wanted key %q in %+v", myKey, entries)
	}
}
Jeromy's avatar
Jeromy committed
372

373
func TestQuerySimple(t *testing.T) { tryAllShardFuncs(t, testQuerySimple) }
374 375

func testBatchPut(dirFunc mkShardFunc, t *testing.T) {
Jeromy's avatar
Jeromy committed
376 377 378
	temp, cleanup := tempdir(t)
	defer cleanup()

379
	fs, err := flatfs.New(temp, dirFunc(2), false)
Jeromy's avatar
Jeromy committed
380 381 382 383
	if err != nil {
		t.Fatalf("New fail: %v\n", err)
	}

Jeromy's avatar
Jeromy committed
384 385
	dstest.RunBatchTest(t, fs)
}
Jeromy's avatar
Jeromy committed
386

387
func TestBatchPut(t *testing.T) { tryAllShardFuncs(t, testBatchPut) }
388 389

func testBatchDelete(dirFunc mkShardFunc, t *testing.T) {
Jeromy's avatar
Jeromy committed
390 391
	temp, cleanup := tempdir(t)
	defer cleanup()
Jeromy's avatar
Jeromy committed
392

393
	fs, err := flatfs.New(temp, dirFunc(2), false)
Jeromy's avatar
Jeromy committed
394
	if err != nil {
Jeromy's avatar
Jeromy committed
395
		t.Fatalf("New fail: %v\n", err)
Jeromy's avatar
Jeromy committed
396 397
	}

Jeromy's avatar
Jeromy committed
398
	dstest.RunBatchDeleteTest(t, fs)
Jeromy's avatar
Jeromy committed
399 400
}

401
func TestBatchDelete(t *testing.T) { tryAllShardFuncs(t, testBatchDelete) }
402

Jeromy's avatar
Jeromy committed
403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
func BenchmarkConsecutivePut(b *testing.B) {
	r := rand.New()
	var blocks [][]byte
	var keys []datastore.Key
	for i := 0; i < b.N; i++ {
		blk := make([]byte, 256*1024)
		r.Read(blk)
		blocks = append(blocks, blk)

		key := base32.StdEncoding.EncodeToString(blk[:8])
		keys = append(keys, datastore.NewKey(key))
	}
	temp, cleanup := tempdir(b)
	defer cleanup()

418
	fs, err := flatfs.New(temp, flatfs.Prefix(2), false)
Jeromy's avatar
Jeromy committed
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
	if err != nil {
		b.Fatalf("New fail: %v\n", err)
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		err := fs.Put(keys[i], blocks[i])
		if err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkBatchedPut(b *testing.B) {
	r := rand.New()
	var blocks [][]byte
	var keys []datastore.Key
	for i := 0; i < b.N; i++ {
		blk := make([]byte, 256*1024)
		r.Read(blk)
		blocks = append(blocks, blk)

		key := base32.StdEncoding.EncodeToString(blk[:8])
		keys = append(keys, datastore.NewKey(key))
	}
	temp, cleanup := tempdir(b)
	defer cleanup()

448
	fs, err := flatfs.New(temp, flatfs.Prefix(2), false)
Jeromy's avatar
Jeromy committed
449 450 451 452 453 454 455
	if err != nil {
		b.Fatalf("New fail: %v\n", err)
	}

	b.ResetTimer()

	for i := 0; i < b.N; {
Jeromy's avatar
Jeromy committed
456 457 458 459
		batch, err := fs.Batch()
		if err != nil {
			b.Fatal(err)
		}
Jeromy's avatar
Jeromy committed
460 461 462 463 464 465 466 467 468 469 470 471 472

		for n := i; i-n < 512 && i < b.N; i++ {
			err := batch.Put(keys[i], blocks[i])
			if err != nil {
				b.Fatal(err)
			}
		}
		err = batch.Commit()
		if err != nil {
			b.Fatal(err)
		}
	}
}