flatfs.go 7.7 KB
Newer Older
Tommi Virtanen's avatar
Tommi Virtanen committed
1 2 3 4 5 6 7 8 9 10
// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"errors"
	"io/ioutil"
	"os"
	"path"
Jeromy's avatar
Jeromy committed
11
	"path/filepath"
Tommi Virtanen's avatar
Tommi Virtanen committed
12
	"strings"
13
	"time"
Tommi Virtanen's avatar
Tommi Virtanen committed
14

Jeromy's avatar
Jeromy committed
15 16
	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/query"
Jakub Sztandera's avatar
Jakub Sztandera committed
17
	"github.com/jbenet/go-os-rename"
18

Jakub Sztandera's avatar
Jakub Sztandera committed
19
	logging "github.com/ipfs/go-log"
Tommi Virtanen's avatar
Tommi Virtanen committed
20 21
)

22 23
var log = logging.Logger("flatfs")

Tommi Virtanen's avatar
Tommi Virtanen committed
24 25 26 27 28 29 30 31 32 33 34
const (
	extension    = ".data"
	maxPrefixLen = 16
)

var (
	ErrBadPrefixLen = errors.New("bad prefix length")
)

type Datastore struct {
	path string
Jeromy's avatar
Jeromy committed
35 36
	// length of the dir splay prefix
	prefixLen int
Jeromy's avatar
Jeromy committed
37 38 39

	// sychronize all writes and directory changes for added safety
	sync bool
Tommi Virtanen's avatar
Tommi Virtanen committed
40 41 42 43
}

var _ datastore.Datastore = (*Datastore)(nil)

Jeromy's avatar
Jeromy committed
44
func New(path string, prefixLen int, sync bool) (*Datastore, error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
45 46 47 48
	if prefixLen <= 0 || prefixLen > maxPrefixLen {
		return nil, ErrBadPrefixLen
	}
	fs := &Datastore{
Jeromy's avatar
Jeromy committed
49 50 51
		path:      path,
		prefixLen: prefixLen,
		sync:      sync,
Tommi Virtanen's avatar
Tommi Virtanen committed
52 53 54 55
	}
	return fs, nil
}

Jeromy's avatar
Jeromy committed
56
var padding = strings.Repeat("_", maxPrefixLen)
Tommi Virtanen's avatar
Tommi Virtanen committed
57 58

func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
59 60
	noslash := key.String()[1:]
	prefix := (noslash + padding)[:fs.prefixLen]
Tommi Virtanen's avatar
Tommi Virtanen committed
61
	dir = path.Join(fs.path, prefix)
62
	file = path.Join(dir, noslash+extension)
Tommi Virtanen's avatar
Tommi Virtanen committed
63 64 65
	return dir, file
}

66 67 68 69 70
func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
Jeromy's avatar
Jeromy committed
71
	return datastore.NewKey(name), true
72 73
}

74
func (fs *Datastore) makePrefixDir(dir string) error {
Jeromy's avatar
Jeromy committed
75 76
	if err := fs.makePrefixDirNoSync(dir); err != nil {
		return err
77 78 79 80 81 82
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
Jeromy's avatar
Jeromy committed
83 84 85 86
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
87 88 89 90
	}
	return nil
}

Jeromy's avatar
Jeromy committed
91 92 93 94 95 96 97 98 99 100 101
func (fs *Datastore) makePrefixDirNoSync(dir string) error {
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

Jeromy's avatar
Jeromy committed
102
var putMaxRetries = 6
103

Tommi Virtanen's avatar
Tommi Virtanen committed
104 105 106 107 108 109
func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

110 111 112 113
	var err error
	for i := 0; i < putMaxRetries; i++ {
		err = fs.doPut(key, val)
		if err == nil {
Jeromy's avatar
Jeromy committed
114
			break
115 116 117
		}

		if !strings.Contains(err.Error(), "too many open files") {
Jeromy's avatar
Jeromy committed
118
			break
119 120
		}

Or Rikon's avatar
Or Rikon committed
121
		log.Errorf("too many open files, retrying in %dms", 100*i)
Jeromy's avatar
Jeromy committed
122
		time.Sleep(time.Millisecond * 100 * time.Duration(i+1))
123 124 125 126 127
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
128
	dir, path := fs.encode(key)
129 130
	if err := fs.makePrefixDir(dir); err != nil {
		return err
Tommi Virtanen's avatar
Tommi Virtanen committed
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
Jeromy's avatar
Jeromy committed
153 154 155 156
	if fs.sync {
		if err := tmp.Sync(); err != nil {
			return err
		}
157
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
158 159 160 161 162
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

163
	err = osrename.Rename(tmp.Name(), path)
Tommi Virtanen's avatar
Tommi Virtanen committed
164 165 166 167 168
	if err != nil {
		return err
	}
	removed = true

Jeromy's avatar
Jeromy committed
169 170 171 172
	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
173
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
174 175 176
	return nil
}

Jeromy's avatar
Jeromy committed
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
		if err := fs.makePrefixDirNoSync(dir); err != nil {
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
Jeromy's avatar
Jeromy committed
222 223 224 225
		if fs.sync {
			if err := fi.Sync(); err != nil {
				return err
			}
Jeromy's avatar
Jeromy committed
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
Jeromy's avatar
Jeromy committed
247 248 249 250 251
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
Jeromy's avatar
Jeromy committed
252 253
		}

Jeromy's avatar
Jeromy committed
254 255 256 257
		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
Jeromy's avatar
Jeromy committed
258 259 260 261 262
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
263 264 265 266 267 268 269 270 271 272 273 274 275 276
func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
277 278 279 280 281 282 283 284 285
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
286 287 288
}

func (fs *Datastore) Delete(key datastore.Key) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
289 290 291 292 293 294 295 296 297
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
298 299 300
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
301 302 303 304 305 306 307 308 309 310 311
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

312
	reschan := make(chan query.Result, query.KeysOnlyBufSize)
Jeromy's avatar
Jeromy committed
313 314 315
	go func() {
		defer close(reschan)
		err := filepath.Walk(fs.path, func(path string, info os.FileInfo, err error) error {
316 317 318
			if os.IsNotExist(err) {
				return nil
			}
319 320 321 322
			if err != nil {
				log.Errorf("Walk func in Query got error: %v", err)
				return err
			}
323

Jeromy's avatar
Jeromy committed
324
			if !info.Mode().IsRegular() || strings.HasPrefix(info.Name(), ".") {
Jeromy's avatar
Jeromy committed
325 326
				return nil
			}
327

Jeromy's avatar
Jeromy committed
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
			key, ok := fs.decode(info.Name())
			if !ok {
				log.Warning("failed to decode entry in flatfs")
				return nil
			}

			reschan <- query.Result{
				Entry: query.Entry{
					Key: key.String(),
				},
			}
			return nil
		})
		if err != nil {
			log.Warning("walk failed: ", err)
343
		}
Jeromy's avatar
Jeromy committed
344 345
	}()
	return query.ResultsWithChan(q, reschan), nil
Tommi Virtanen's avatar
Tommi Virtanen committed
346 347
}

Jeromy's avatar
Jeromy committed
348 349 350 351
func (fs *Datastore) Close() error {
	return nil
}

Jeromy's avatar
Jeromy committed
352
type flatfsBatch struct {
Jeromy's avatar
Jeromy committed
353 354 355 356 357 358
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

Jeromy's avatar
Jeromy committed
359
func (fs *Datastore) Batch() (datastore.Batch, error) {
Jeromy's avatar
Jeromy committed
360
	return &flatfsBatch{
Jeromy's avatar
Jeromy committed
361 362 363
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
Jeromy's avatar
Jeromy committed
364
	}, nil
Jeromy's avatar
Jeromy committed
365 366
}

Jeromy's avatar
Jeromy committed
367
func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
Jeromy's avatar
Jeromy committed
368 369 370 371
	bt.puts[key] = val
	return nil
}

Jeromy's avatar
Jeromy committed
372
func (bt *flatfsBatch) Delete(key datastore.Key) error {
Jeromy's avatar
Jeromy committed
373 374 375 376
	bt.deletes[key] = struct{}{}
	return nil
}

Jeromy's avatar
Jeromy committed
377
func (bt *flatfsBatch) Commit() error {
Jeromy's avatar
Jeromy committed
378 379 380 381 382 383 384 385 386 387 388 389 390
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
391 392 393
var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}