flatfs.go 9.61 KB
Newer Older
Tommi Virtanen's avatar
Tommi Virtanen committed
1 2 3 4 5 6 7
// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"errors"
8
	"fmt"
Tommi Virtanen's avatar
Tommi Virtanen committed
9 10 11
	"io/ioutil"
	"os"
	"path"
Jeromy's avatar
Jeromy committed
12
	"path/filepath"
Tommi Virtanen's avatar
Tommi Virtanen committed
13
	"strings"
14
	"time"
Tommi Virtanen's avatar
Tommi Virtanen committed
15

Jeromy's avatar
Jeromy committed
16 17
	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/query"
Jakub Sztandera's avatar
Jakub Sztandera committed
18
	"github.com/jbenet/go-os-rename"
19

Jakub Sztandera's avatar
Jakub Sztandera committed
20
	logging "github.com/ipfs/go-log"
Tommi Virtanen's avatar
Tommi Virtanen committed
21 22
)

23 24
var log = logging.Logger("flatfs")

Tommi Virtanen's avatar
Tommi Virtanen committed
25
const (
26
	extension = ".data"
Tommi Virtanen's avatar
Tommi Virtanen committed
27 28 29 30
)

type Datastore struct {
	path string
31

32 33
	shardFunc string
	getDir    ShardFunc
Jeromy's avatar
Jeromy committed
34 35 36

	// sychronize all writes and directory changes for added safety
	sync bool
Tommi Virtanen's avatar
Tommi Virtanen committed
37 38 39 40
}

var _ datastore.Datastore = (*Datastore)(nil)

41 42
type ShardFunc func(string) string

Kevin Atkinson's avatar
Kevin Atkinson committed
43
var (
44 45
	ErrDatastoreExists       = errors.New("datastore already exist")
	ErrDatastoreDoesNotExist = errors.New("datastore directory does not exist")
46
	ErrShardingFileMissing   = fmt.Errorf("%s file not found in datastore", SHARDING_FN)
Kevin Atkinson's avatar
Kevin Atkinson committed
47
)
48

49
const IPFS_DEF_SHARD = "/repo/flatfs/shard/v1/next-to-last/2"
50

51
func Create(path string, funStr string) error {
52

53 54 55
	err := os.Mkdir(path, 0777)
	if err != nil && !os.IsExist(err) {
		return err
Tommi Virtanen's avatar
Tommi Virtanen committed
56 57
	}

58 59 60 61 62
	fun, err := ParseShardFunc(funStr)
	if err != nil {
		return err
	}

63 64
	dsFun, err := ReadShardFunc(path)
	switch err {
Kevin Atkinson's avatar
Kevin Atkinson committed
65
	case ErrShardingFileMissing:
66 67 68 69 70
		isEmpty, err := DirIsEmpty(path)
		if err != nil {
			return err
		}
		if !isEmpty {
71
			return fmt.Errorf("directory missing %s file: %s", SHARDING_FN, path)
72 73 74
		}

		err = WriteShardFunc(path, fun)
75 76 77
		if err != nil {
			return err
		}
78 79
		err = WriteReadme(path, fun)
		return err
80
	case nil:
81
		if fun.String() != dsFun.String() {
82
			return fmt.Errorf("specified shard func '%s' does not match repo shard func '%s'",
83
				fun.String(), dsFun.String())
84
		}
Kevin Atkinson's avatar
Kevin Atkinson committed
85
		return ErrDatastoreExists
86
	default:
87
		return err
88
	}
89 90 91 92 93
}

func Open(path string, sync bool) (*Datastore, error) {
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
Kevin Atkinson's avatar
Kevin Atkinson committed
94
		return nil, ErrDatastoreDoesNotExist
95 96 97 98
	} else if err != nil {
		return nil, err
	}

99
	shardId, err := ReadShardFunc(path)
100
	if err != nil {
101
		return nil, err
102 103
	}

104 105
	fs := &Datastore{
		path:      path,
106 107
		shardFunc: shardId.String(),
		getDir:    shardId.Func(),
108
		sync:      sync,
109
	}
110 111 112
	return fs, nil
}

113
// convenience method
114 115
func CreateOrOpen(path string, fun string, sync bool) (*Datastore, error) {
	err := Create(path, fun)
Kevin Atkinson's avatar
Kevin Atkinson committed
116
	if err != nil && err != ErrDatastoreExists {
117 118 119 120 121
		return nil, err
	}
	return Open(path, sync)
}

122 123
func (fs *Datastore) ShardFunc() string {
	return fs.shardFunc
124 125
}

Tommi Virtanen's avatar
Tommi Virtanen committed
126
func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
127
	noslash := key.String()[1:]
128
	dir = path.Join(fs.path, fs.getDir(noslash))
129
	file = path.Join(dir, noslash+extension)
Tommi Virtanen's avatar
Tommi Virtanen committed
130 131 132
	return dir, file
}

133 134 135 136 137
func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
Jeromy's avatar
Jeromy committed
138
	return datastore.NewKey(name), true
139 140
}

141 142
func (fs *Datastore) makeDir(dir string) error {
	if err := fs.makeDirNoSync(dir); err != nil {
Jeromy's avatar
Jeromy committed
143
		return err
144 145 146 147 148 149
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
Jeromy's avatar
Jeromy committed
150 151 152 153
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
154 155 156 157
	}
	return nil
}

158
func (fs *Datastore) makeDirNoSync(dir string) error {
Jeromy's avatar
Jeromy committed
159 160 161 162 163 164 165 166 167 168
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

Jeromy's avatar
Jeromy committed
169
var putMaxRetries = 6
170

Tommi Virtanen's avatar
Tommi Virtanen committed
171 172 173 174 175 176
func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

177
	var err error
Jeromy's avatar
Jeromy committed
178
	for i := 1; i <= putMaxRetries; i++ {
179 180
		err = fs.doPut(key, val)
		if err == nil {
Jeromy's avatar
Jeromy committed
181
			break
182 183 184
		}

		if !strings.Contains(err.Error(), "too many open files") {
Jeromy's avatar
Jeromy committed
185
			break
186 187
		}

Or Rikon's avatar
Or Rikon committed
188
		log.Errorf("too many open files, retrying in %dms", 100*i)
Jeromy's avatar
Jeromy committed
189
		time.Sleep(time.Millisecond * 100 * time.Duration(i))
190 191 192 193 194
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
195
	dir, path := fs.encode(key)
196
	if err := fs.makeDir(dir); err != nil {
197
		return err
Tommi Virtanen's avatar
Tommi Virtanen committed
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
Jeromy's avatar
Jeromy committed
220
	if fs.sync {
221
		if err := syncFile(tmp); err != nil {
Jeromy's avatar
Jeromy committed
222 223
			return err
		}
224
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
225 226 227 228 229
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

230
	err = osrename.Rename(tmp.Name(), path)
Tommi Virtanen's avatar
Tommi Virtanen committed
231 232 233 234 235
	if err != nil {
		return err
	}
	removed = true

Jeromy's avatar
Jeromy committed
236 237 238 239
	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
240
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
241 242 243
	return nil
}

Jeromy's avatar
Jeromy committed
244 245 246 247 248 249 250 251 252 253
func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
254
		if err := fs.makeDirNoSync(dir); err != nil {
Jeromy's avatar
Jeromy committed
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
Jeromy's avatar
Jeromy committed
289
		if fs.sync {
290
			if err := syncFile(fi); err != nil {
Jeromy's avatar
Jeromy committed
291 292
				return err
			}
Jeromy's avatar
Jeromy committed
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
Jeromy's avatar
Jeromy committed
314 315 316 317 318
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
Jeromy's avatar
Jeromy committed
319 320
		}

Jeromy's avatar
Jeromy committed
321 322 323 324
		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
Jeromy's avatar
Jeromy committed
325 326 327 328 329
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
330 331 332 333 334 335 336 337 338 339 340 341 342 343
func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
344 345 346 347 348 349 350 351 352
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
353 354 355
}

func (fs *Datastore) Delete(key datastore.Key) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
356 357 358 359 360 361 362 363 364
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
365 366 367
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
368 369 370 371 372 373 374 375 376 377 378
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

379
	reschan := make(chan query.Result, query.KeysOnlyBufSize)
Jeromy's avatar
Jeromy committed
380 381
	go func() {
		defer close(reschan)
382
		err := fs.walkTopLevel(fs.path, reschan)
Jeromy's avatar
Jeromy committed
383
		if err != nil {
384
			reschan <- query.Result{Error: errors.New("walk failed: " + err.Error())}
385
		}
Jeromy's avatar
Jeromy committed
386 387
	}()
	return query.ResultsWithChan(q, reschan), nil
Tommi Virtanen's avatar
Tommi Virtanen committed
388 389
}

390 391 392 393 394
func (fs *Datastore) walkTopLevel(path string, reschan chan query.Result) error {
	dir, err := os.Open(path)
	if err != nil {
		return err
	}
Kevin Atkinson's avatar
Kevin Atkinson committed
395
	defer dir.Close()
396 397 398 399 400
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, dir := range names {
Kevin Atkinson's avatar
Kevin Atkinson committed
401 402 403 404 405

		if len(dir) == 0 || dir[0] == '.' {
			continue
		}

406 407 408 409
		err = fs.walk(filepath.Join(path, dir), reschan)
		if err != nil {
			return err
		}
Kevin Atkinson's avatar
Kevin Atkinson committed
410

411 412 413 414 415 416 417 418 419
	}
	return nil
}

func (fs *Datastore) walk(path string, reschan chan query.Result) error {
	dir, err := os.Open(path)
	if err != nil {
		return err
	}
Kevin Atkinson's avatar
Kevin Atkinson committed
420
	defer dir.Close()
421 422 423 424 425 426 427 428 429 430

	// ignore non-directories
	fileInfo, err := dir.Stat()
	if err != nil {
		return err
	}
	if !fileInfo.IsDir() {
		return nil
	}

431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, fn := range names {

		if len(fn) == 0 || fn[0] == '.' {
			continue
		}

		key, ok := fs.decode(fn)
		if !ok {
			log.Warning("failed to decode entry in flatfs")
			continue
		}

		reschan <- query.Result{
			Entry: query.Entry{
				Key: key.String(),
			},
		}
	}
	return nil
}

Jeromy's avatar
Jeromy committed
456 457 458 459
func (fs *Datastore) Close() error {
	return nil
}

Jeromy's avatar
Jeromy committed
460
type flatfsBatch struct {
Jeromy's avatar
Jeromy committed
461 462 463 464 465 466
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

Jeromy's avatar
Jeromy committed
467
func (fs *Datastore) Batch() (datastore.Batch, error) {
Jeromy's avatar
Jeromy committed
468
	return &flatfsBatch{
Jeromy's avatar
Jeromy committed
469 470 471
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
Jeromy's avatar
Jeromy committed
472
	}, nil
Jeromy's avatar
Jeromy committed
473 474
}

Jeromy's avatar
Jeromy committed
475
func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
Jeromy's avatar
Jeromy committed
476 477 478 479
	bt.puts[key] = val
	return nil
}

Jeromy's avatar
Jeromy committed
480
func (bt *flatfsBatch) Delete(key datastore.Key) error {
Jeromy's avatar
Jeromy committed
481 482 483 484
	bt.deletes[key] = struct{}{}
	return nil
}

Jeromy's avatar
Jeromy committed
485
func (bt *flatfsBatch) Commit() error {
Jeromy's avatar
Jeromy committed
486 487 488 489 490 491 492 493 494 495 496 497 498
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
499 500 501
var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}