flatfs.go 8.02 KB
Newer Older
Tommi Virtanen's avatar
Tommi Virtanen committed
1 2 3 4 5 6 7 8 9 10
// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"errors"
	"io/ioutil"
	"os"
	"path"
Jeromy's avatar
Jeromy committed
11
	"path/filepath"
Tommi Virtanen's avatar
Tommi Virtanen committed
12
	"strings"
13
	"time"
Tommi Virtanen's avatar
Tommi Virtanen committed
14

Jeromy's avatar
Jeromy committed
15 16
	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/query"
Jakub Sztandera's avatar
Jakub Sztandera committed
17
	"github.com/jbenet/go-os-rename"
18

Jakub Sztandera's avatar
Jakub Sztandera committed
19
	logging "github.com/ipfs/go-log"
Tommi Virtanen's avatar
Tommi Virtanen committed
20 21
)

22 23
var log = logging.Logger("flatfs")

Tommi Virtanen's avatar
Tommi Virtanen committed
24 25 26 27 28 29 30 31 32 33 34
const (
	extension    = ".data"
	maxPrefixLen = 16
)

var (
	ErrBadPrefixLen = errors.New("bad prefix length")
)

type Datastore struct {
	path string
Jeromy's avatar
Jeromy committed
35 36
	// length of the dir splay prefix
	prefixLen int
Jeromy's avatar
Jeromy committed
37 38 39

	// sychronize all writes and directory changes for added safety
	sync bool
Tommi Virtanen's avatar
Tommi Virtanen committed
40 41 42 43
}

var _ datastore.Datastore = (*Datastore)(nil)

Jeromy's avatar
Jeromy committed
44
func New(path string, prefixLen int, sync bool) (*Datastore, error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
45 46 47 48
	if prefixLen <= 0 || prefixLen > maxPrefixLen {
		return nil, ErrBadPrefixLen
	}
	fs := &Datastore{
Jeromy's avatar
Jeromy committed
49 50 51
		path:      path,
		prefixLen: prefixLen,
		sync:      sync,
Tommi Virtanen's avatar
Tommi Virtanen committed
52 53 54 55
	}
	return fs, nil
}

Jeromy's avatar
Jeromy committed
56
var padding = strings.Repeat("_", maxPrefixLen)
Tommi Virtanen's avatar
Tommi Virtanen committed
57 58

func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
59 60
	noslash := key.String()[1:]
	prefix := (noslash + padding)[:fs.prefixLen]
Tommi Virtanen's avatar
Tommi Virtanen committed
61
	dir = path.Join(fs.path, prefix)
62
	file = path.Join(dir, noslash+extension)
Tommi Virtanen's avatar
Tommi Virtanen committed
63 64 65
	return dir, file
}

66 67 68 69 70
func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
Jeromy's avatar
Jeromy committed
71
	return datastore.NewKey(name), true
72 73
}

74
func (fs *Datastore) makePrefixDir(dir string) error {
Jeromy's avatar
Jeromy committed
75 76
	if err := fs.makePrefixDirNoSync(dir); err != nil {
		return err
77 78 79 80 81 82
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
Jeromy's avatar
Jeromy committed
83 84 85 86
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
87 88 89 90
	}
	return nil
}

Jeromy's avatar
Jeromy committed
91 92 93 94 95 96 97 98 99 100 101
func (fs *Datastore) makePrefixDirNoSync(dir string) error {
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

Jeromy's avatar
Jeromy committed
102
var putMaxRetries = 6
103

Tommi Virtanen's avatar
Tommi Virtanen committed
104 105 106 107 108 109
func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

110
	var err error
Jeromy's avatar
Jeromy committed
111
	for i := 1; i <= putMaxRetries; i++ {
112 113
		err = fs.doPut(key, val)
		if err == nil {
Jeromy's avatar
Jeromy committed
114
			break
115 116 117
		}

		if !strings.Contains(err.Error(), "too many open files") {
Jeromy's avatar
Jeromy committed
118
			break
119 120
		}

Or Rikon's avatar
Or Rikon committed
121
		log.Errorf("too many open files, retrying in %dms", 100*i)
Jeromy's avatar
Jeromy committed
122
		time.Sleep(time.Millisecond * 100 * time.Duration(i))
123 124 125 126 127
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
128
	dir, path := fs.encode(key)
129 130
	if err := fs.makePrefixDir(dir); err != nil {
		return err
Tommi Virtanen's avatar
Tommi Virtanen committed
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
Jeromy's avatar
Jeromy committed
153
	if fs.sync {
154
		if err := syncFile(tmp); err != nil {
Jeromy's avatar
Jeromy committed
155 156
			return err
		}
157
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
158 159 160 161 162
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

163
	err = osrename.Rename(tmp.Name(), path)
Tommi Virtanen's avatar
Tommi Virtanen committed
164 165 166 167 168
	if err != nil {
		return err
	}
	removed = true

Jeromy's avatar
Jeromy committed
169 170 171 172
	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
173
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
174 175 176
	return nil
}

Jeromy's avatar
Jeromy committed
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
		if err := fs.makePrefixDirNoSync(dir); err != nil {
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
Jeromy's avatar
Jeromy committed
222
		if fs.sync {
223
			if err := syncFile(fi); err != nil {
Jeromy's avatar
Jeromy committed
224 225
				return err
			}
Jeromy's avatar
Jeromy committed
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
Jeromy's avatar
Jeromy committed
247 248 249 250 251
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
Jeromy's avatar
Jeromy committed
252 253
		}

Jeromy's avatar
Jeromy committed
254 255 256 257
		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
Jeromy's avatar
Jeromy committed
258 259 260 261 262
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
263 264 265 266 267 268 269 270 271 272 273 274 275 276
func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
277 278 279 280 281 282 283 284 285
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
286 287 288
}

func (fs *Datastore) Delete(key datastore.Key) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
289 290 291 292 293 294 295 296 297
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
298 299 300
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
301 302 303 304 305 306 307 308 309 310 311
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

312
	reschan := make(chan query.Result, query.KeysOnlyBufSize)
Jeromy's avatar
Jeromy committed
313 314
	go func() {
		defer close(reschan)
315
		err := fs.walkTopLevel(fs.path, reschan)
Jeromy's avatar
Jeromy committed
316 317
		if err != nil {
			log.Warning("walk failed: ", err)
318
		}
Jeromy's avatar
Jeromy committed
319 320
	}()
	return query.ResultsWithChan(q, reschan), nil
Tommi Virtanen's avatar
Tommi Virtanen committed
321 322
}

323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
func (fs *Datastore) walkTopLevel(path string, reschan chan query.Result) error {
	dir, err := os.Open(path)
	if err != nil {
		return err
	}
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, dir := range names {
		err = fs.walk(filepath.Join(path, dir), reschan)
		if err != nil {
			return err
		}
	}
	return nil
}

func (fs *Datastore) walk(path string, reschan chan query.Result) error {
	dir, err := os.Open(path)
	if err != nil {
		return err
	}
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, fn := range names {

		if len(fn) == 0 || fn[0] == '.' {
			continue
		}

		key, ok := fs.decode(fn)
		if !ok {
			log.Warning("failed to decode entry in flatfs")
			continue
		}

		reschan <- query.Result{
			Entry: query.Entry{
				Key: key.String(),
			},
		}
	}
	return nil
}

Jeromy's avatar
Jeromy committed
371 372 373 374
func (fs *Datastore) Close() error {
	return nil
}

Jeromy's avatar
Jeromy committed
375
type flatfsBatch struct {
Jeromy's avatar
Jeromy committed
376 377 378 379 380 381
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

Jeromy's avatar
Jeromy committed
382
func (fs *Datastore) Batch() (datastore.Batch, error) {
Jeromy's avatar
Jeromy committed
383
	return &flatfsBatch{
Jeromy's avatar
Jeromy committed
384 385 386
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
Jeromy's avatar
Jeromy committed
387
	}, nil
Jeromy's avatar
Jeromy committed
388 389
}

Jeromy's avatar
Jeromy committed
390
func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
Jeromy's avatar
Jeromy committed
391 392 393 394
	bt.puts[key] = val
	return nil
}

Jeromy's avatar
Jeromy committed
395
func (bt *flatfsBatch) Delete(key datastore.Key) error {
Jeromy's avatar
Jeromy committed
396 397 398 399
	bt.deletes[key] = struct{}{}
	return nil
}

Jeromy's avatar
Jeromy committed
400
func (bt *flatfsBatch) Commit() error {
Jeromy's avatar
Jeromy committed
401 402 403 404 405 406 407 408 409 410 411 412 413
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
414 415 416
var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}