flatfs.go 8.26 KB
Newer Older
Tommi Virtanen's avatar
Tommi Virtanen committed
1 2 3 4 5 6 7 8 9 10 11 12
// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"encoding/hex"
	"errors"
	"io/ioutil"
	"os"
	"path"
	"strings"
13
	"time"
Tommi Virtanen's avatar
Tommi Virtanen committed
14 15

	"github.com/jbenet/go-datastore"
16
	"github.com/jbenet/go-datastore/Godeps/_workspace/src/github.com/jbenet/go-os-rename"
Tommi Virtanen's avatar
Tommi Virtanen committed
17
	"github.com/jbenet/go-datastore/query"
18 19

	logging "github.com/jbenet/go-datastore/Godeps/_workspace/src/github.com/ipfs/go-log"
Tommi Virtanen's avatar
Tommi Virtanen committed
20 21
)

22 23
var log = logging.Logger("flatfs")

Tommi Virtanen's avatar
Tommi Virtanen committed
24 25 26 27 28 29 30 31 32 33 34 35 36
const (
	extension    = ".data"
	maxPrefixLen = 16
)

var (
	ErrBadPrefixLen = errors.New("bad prefix length")
)

type Datastore struct {
	path string
	// length of the dir splay prefix, in bytes of hex digits
	hexPrefixLen int
Jeromy's avatar
Jeromy committed
37 38 39

	// sychronize all writes and directory changes for added safety
	sync bool
Tommi Virtanen's avatar
Tommi Virtanen committed
40 41 42 43
}

var _ datastore.Datastore = (*Datastore)(nil)

Jeromy's avatar
Jeromy committed
44
func New(path string, prefixLen int, sync bool) (*Datastore, error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
45 46 47 48 49 50 51 52 53 54 55 56 57 58
	if prefixLen <= 0 || prefixLen > maxPrefixLen {
		return nil, ErrBadPrefixLen
	}
	fs := &Datastore{
		path: path,
		// convert from binary bytes to bytes of hex encoding
		hexPrefixLen: prefixLen * hex.EncodedLen(1),
	}
	return fs, nil
}

var padding = strings.Repeat("_", maxPrefixLen*hex.EncodedLen(1))

func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
59
	safe := hex.EncodeToString(key.Bytes()[1:])
Tommi Virtanen's avatar
Tommi Virtanen committed
60 61 62 63 64 65
	prefix := (safe + padding)[:fs.hexPrefixLen]
	dir = path.Join(fs.path, prefix)
	file = path.Join(dir, safe+extension)
	return dir, file
}

66 67 68 69 70 71 72 73 74 75 76 77
func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
	k, err := hex.DecodeString(name)
	if err != nil {
		return datastore.Key{}, false
	}
	return datastore.NewKey(string(k)), true
}

78
func (fs *Datastore) makePrefixDir(dir string) error {
Jeromy's avatar
Jeromy committed
79 80
	if err := fs.makePrefixDirNoSync(dir); err != nil {
		return err
81 82 83 84 85 86
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
Jeromy's avatar
Jeromy committed
87 88 89 90
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
91 92 93 94
	}
	return nil
}

Jeromy's avatar
Jeromy committed
95 96 97 98 99 100 101 102 103 104 105
func (fs *Datastore) makePrefixDirNoSync(dir string) error {
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

106 107
var putMaxRetries = 3

Tommi Virtanen's avatar
Tommi Virtanen committed
108 109 110 111 112 113
func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
	var err error
	for i := 0; i < putMaxRetries; i++ {
		err = fs.doPut(key, val)
		if err == nil {
			return nil
		}

		if !strings.Contains(err.Error(), "too many open files") {
			return err
		}

		log.Error("too many open files, retrying in %dms", 100*i)
		time.Sleep(time.Millisecond * 100 * time.Duration(i))
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
132
	dir, path := fs.encode(key)
133 134
	if err := fs.makePrefixDir(dir); err != nil {
		return err
Tommi Virtanen's avatar
Tommi Virtanen committed
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
Jeromy's avatar
Jeromy committed
157 158 159 160
	if fs.sync {
		if err := tmp.Sync(); err != nil {
			return err
		}
161
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
162 163 164 165 166
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

167
	err = osrename.Rename(tmp.Name(), path)
Tommi Virtanen's avatar
Tommi Virtanen committed
168 169 170 171 172
	if err != nil {
		return err
	}
	removed = true

Jeromy's avatar
Jeromy committed
173 174 175 176
	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
177
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
178 179 180
	return nil
}

Jeromy's avatar
Jeromy committed
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
		if err := fs.makePrefixDirNoSync(dir); err != nil {
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
Jeromy's avatar
Jeromy committed
226 227 228 229
		if fs.sync {
			if err := fi.Sync(); err != nil {
				return err
			}
Jeromy's avatar
Jeromy committed
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
Jeromy's avatar
Jeromy committed
251 252 253 254 255
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
Jeromy's avatar
Jeromy committed
256 257
		}

Jeromy's avatar
Jeromy committed
258 259 260 261
		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
Jeromy's avatar
Jeromy committed
262 263 264 265 266
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
267 268 269 270 271 272 273 274 275 276 277 278 279 280
func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
281 282 283 284 285 286 287 288 289
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
290 291 292
}

func (fs *Datastore) Delete(key datastore.Key) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
293 294 295 296 297 298 299 300 301
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
302 303 304
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

	// TODO this dumb implementation gathers all keys into a single slice.
	root, err := os.Open(fs.path)
	if err != nil {
		return nil, err
	}
	defer root.Close()

	var res []query.Entry
	prefixes, err := root.Readdir(0)
	if err != nil {
		return nil, err
	}
	for _, fi := range prefixes {
329 330
		var err error
		res, err = fs.enumerateKeys(fi, res)
331 332 333
		if err != nil {
			return nil, err
		}
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
	}
	return query.ResultsWithEntries(q, res), nil
}

func (fs *Datastore) enumerateKeys(fi os.FileInfo, res []query.Entry) ([]query.Entry, error) {
	if !fi.IsDir() || fi.Name()[0] == '.' {
		return res, nil
	}
	child, err := os.Open(path.Join(fs.path, fi.Name()))
	if err != nil {
		return nil, err
	}
	defer child.Close()
	objs, err := child.Readdir(0)
	if err != nil {
		return nil, err
	}
	for _, fi := range objs {
		if !fi.Mode().IsRegular() || fi.Name()[0] == '.' {
			return res, nil
354
		}
355 356 357
		key, ok := fs.decode(fi.Name())
		if !ok {
			return res, nil
358
		}
359
		res = append(res, query.Entry{Key: key.String()})
360
	}
361
	return res, nil
Tommi Virtanen's avatar
Tommi Virtanen committed
362 363
}

Jeromy's avatar
Jeromy committed
364 365 366 367
func (fs *Datastore) Close() error {
	return nil
}

Jeromy's avatar
Jeromy committed
368
type flatfsBatch struct {
Jeromy's avatar
Jeromy committed
369 370 371 372 373 374
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

Jeromy's avatar
Jeromy committed
375
func (fs *Datastore) Batch() (datastore.Batch, error) {
Jeromy's avatar
Jeromy committed
376
	return &flatfsBatch{
Jeromy's avatar
Jeromy committed
377 378 379
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
Jeromy's avatar
Jeromy committed
380
	}, nil
Jeromy's avatar
Jeromy committed
381 382
}

Jeromy's avatar
Jeromy committed
383
func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
Jeromy's avatar
Jeromy committed
384 385 386 387
	bt.puts[key] = val
	return nil
}

Jeromy's avatar
Jeromy committed
388
func (bt *flatfsBatch) Delete(key datastore.Key) error {
Jeromy's avatar
Jeromy committed
389 390 391 392
	bt.deletes[key] = struct{}{}
	return nil
}

Jeromy's avatar
Jeromy committed
393
func (bt *flatfsBatch) Commit() error {
Jeromy's avatar
Jeromy committed
394 395 396 397 398 399 400 401 402 403 404 405 406
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
407 408 409
var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}