flatfs.go 7.78 KB
Newer Older
Tommi Virtanen's avatar
Tommi Virtanen committed
1 2 3 4 5 6 7 8 9 10 11
// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
	"encoding/hex"
	"errors"
	"io/ioutil"
	"os"
	"path"
Jeromy's avatar
Jeromy committed
12
	"path/filepath"
Tommi Virtanen's avatar
Tommi Virtanen committed
13
	"strings"
14
	"time"
Tommi Virtanen's avatar
Tommi Virtanen committed
15

Jeromy's avatar
Jeromy committed
16
	"github.com/ipfs/go-datastore"
Jakub Sztandera's avatar
Jakub Sztandera committed
17
	"github.com/jbenet/go-os-rename"
Jeromy's avatar
Jeromy committed
18
	"github.com/ipfs/go-datastore/query"
19

Jakub Sztandera's avatar
Jakub Sztandera committed
20
	logging "github.com/ipfs/go-log"
Tommi Virtanen's avatar
Tommi Virtanen committed
21 22
)

23 24
var log = logging.Logger("flatfs")

Tommi Virtanen's avatar
Tommi Virtanen committed
25 26 27 28 29 30 31 32 33 34 35 36 37
const (
	extension    = ".data"
	maxPrefixLen = 16
)

var (
	ErrBadPrefixLen = errors.New("bad prefix length")
)

type Datastore struct {
	path string
	// length of the dir splay prefix, in bytes of hex digits
	hexPrefixLen int
Jeromy's avatar
Jeromy committed
38 39 40

	// sychronize all writes and directory changes for added safety
	sync bool
Tommi Virtanen's avatar
Tommi Virtanen committed
41 42 43 44
}

var _ datastore.Datastore = (*Datastore)(nil)

Jeromy's avatar
Jeromy committed
45
func New(path string, prefixLen int, sync bool) (*Datastore, error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
46 47 48 49 50 51 52
	if prefixLen <= 0 || prefixLen > maxPrefixLen {
		return nil, ErrBadPrefixLen
	}
	fs := &Datastore{
		path: path,
		// convert from binary bytes to bytes of hex encoding
		hexPrefixLen: prefixLen * hex.EncodedLen(1),
53
		sync:         sync,
Tommi Virtanen's avatar
Tommi Virtanen committed
54 55 56 57 58 59 60
	}
	return fs, nil
}

var padding = strings.Repeat("_", maxPrefixLen*hex.EncodedLen(1))

func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
61
	safe := hex.EncodeToString(key.Bytes()[1:])
Tommi Virtanen's avatar
Tommi Virtanen committed
62 63 64 65 66 67
	prefix := (safe + padding)[:fs.hexPrefixLen]
	dir = path.Join(fs.path, prefix)
	file = path.Join(dir, safe+extension)
	return dir, file
}

68 69 70 71 72 73 74 75 76 77 78 79
func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
	if path.Ext(file) != extension {
		return datastore.Key{}, false
	}
	name := file[:len(file)-len(extension)]
	k, err := hex.DecodeString(name)
	if err != nil {
		return datastore.Key{}, false
	}
	return datastore.NewKey(string(k)), true
}

80
func (fs *Datastore) makePrefixDir(dir string) error {
Jeromy's avatar
Jeromy committed
81 82
	if err := fs.makePrefixDirNoSync(dir); err != nil {
		return err
83 84 85 86 87 88
	}

	// In theory, if we create a new prefix dir and add a file to
	// it, the creation of the prefix dir itself might not be
	// durable yet. Sync the root dir after a successful mkdir of
	// a prefix dir, just to be paranoid.
Jeromy's avatar
Jeromy committed
89 90 91 92
	if fs.sync {
		if err := syncDir(fs.path); err != nil {
			return err
		}
93 94 95 96
	}
	return nil
}

Jeromy's avatar
Jeromy committed
97 98 99 100 101 102 103 104 105 106 107
func (fs *Datastore) makePrefixDirNoSync(dir string) error {
	if err := os.Mkdir(dir, 0777); err != nil {
		// EEXIST is safe to ignore here, that just means the prefix
		// directory already existed.
		if !os.IsExist(err) {
			return err
		}
	}
	return nil
}

108 109
var putMaxRetries = 3

Tommi Virtanen's avatar
Tommi Virtanen committed
110 111 112 113 114 115
func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
	val, ok := value.([]byte)
	if !ok {
		return datastore.ErrInvalidType
	}

116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
	var err error
	for i := 0; i < putMaxRetries; i++ {
		err = fs.doPut(key, val)
		if err == nil {
			return nil
		}

		if !strings.Contains(err.Error(), "too many open files") {
			return err
		}

		log.Error("too many open files, retrying in %dms", 100*i)
		time.Sleep(time.Millisecond * 100 * time.Duration(i))
	}
	return err
}

func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
134
	dir, path := fs.encode(key)
135 136
	if err := fs.makePrefixDir(dir); err != nil {
		return err
Tommi Virtanen's avatar
Tommi Virtanen committed
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
	}

	tmp, err := ioutil.TempFile(dir, "put-")
	if err != nil {
		return err
	}
	closed := false
	removed := false
	defer func() {
		if !closed {
			// silence errcheck
			_ = tmp.Close()
		}
		if !removed {
			// silence errcheck
			_ = os.Remove(tmp.Name())
		}
	}()

	if _, err := tmp.Write(val); err != nil {
		return err
	}
Jeromy's avatar
Jeromy committed
159 160 161 162
	if fs.sync {
		if err := tmp.Sync(); err != nil {
			return err
		}
163
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
164 165 166 167 168
	if err := tmp.Close(); err != nil {
		return err
	}
	closed = true

169
	err = osrename.Rename(tmp.Name(), path)
Tommi Virtanen's avatar
Tommi Virtanen committed
170 171 172 173 174
	if err != nil {
		return err
	}
	removed = true

Jeromy's avatar
Jeromy committed
175 176 177 178
	if fs.sync {
		if err := syncDir(dir); err != nil {
			return err
		}
179
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
180 181 182
	return nil
}

Jeromy's avatar
Jeromy committed
183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
	var dirsToSync []string
	files := make(map[*os.File]string)

	for key, value := range data {
		val, ok := value.([]byte)
		if !ok {
			return datastore.ErrInvalidType
		}
		dir, path := fs.encode(key)
		if err := fs.makePrefixDirNoSync(dir); err != nil {
			return err
		}
		dirsToSync = append(dirsToSync, dir)

		tmp, err := ioutil.TempFile(dir, "put-")
		if err != nil {
			return err
		}

		if _, err := tmp.Write(val); err != nil {
			return err
		}

		files[tmp] = path
	}

	ops := make(map[*os.File]int)

	defer func() {
		for fi, _ := range files {
			val, _ := ops[fi]
			switch val {
			case 0:
				_ = fi.Close()
				fallthrough
			case 1:
				_ = os.Remove(fi.Name())
			}
		}
	}()

	// Now we sync everything
	// sync and close files
	for fi, _ := range files {
Jeromy's avatar
Jeromy committed
228 229 230 231
		if fs.sync {
			if err := fi.Sync(); err != nil {
				return err
			}
Jeromy's avatar
Jeromy committed
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
		}

		if err := fi.Close(); err != nil {
			return err
		}

		// signify closed
		ops[fi] = 1
	}

	// move files to their proper places
	for fi, path := range files {
		if err := osrename.Rename(fi.Name(), path); err != nil {
			return err
		}

		// signify removed
		ops[fi] = 2
	}

	// now sync the dirs for those files
Jeromy's avatar
Jeromy committed
253 254 255 256 257
	if fs.sync {
		for _, dir := range dirsToSync {
			if err := syncDir(dir); err != nil {
				return err
			}
Jeromy's avatar
Jeromy committed
258 259
		}

Jeromy's avatar
Jeromy committed
260 261 262 263
		// sync top flatfs dir
		if err := syncDir(fs.path); err != nil {
			return err
		}
Jeromy's avatar
Jeromy committed
264 265 266 267 268
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
269 270 271 272 273 274 275 276 277 278 279 280 281 282
func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
	_, path := fs.encode(key)
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, datastore.ErrNotFound
		}
		// no specific error to return, so just pass it through
		return nil, err
	}
	return data, nil
}

func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
Tommi Virtanen's avatar
Tommi Virtanen committed
283 284 285 286 287 288 289 290 291
	_, path := fs.encode(key)
	switch _, err := os.Stat(path); {
	case err == nil:
		return true, nil
	case os.IsNotExist(err):
		return false, nil
	default:
		return false, err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
292 293 294
}

func (fs *Datastore) Delete(key datastore.Key) error {
Tommi Virtanen's avatar
Tommi Virtanen committed
295 296 297 298 299 300 301 302 303
	_, path := fs.encode(key)
	switch err := os.Remove(path); {
	case err == nil:
		return nil
	case os.IsNotExist(err):
		return datastore.ErrNotFound
	default:
		return err
	}
Tommi Virtanen's avatar
Tommi Virtanen committed
304 305 306
}

func (fs *Datastore) Query(q query.Query) (query.Results, error) {
307 308 309 310 311 312 313 314 315 316 317
	if (q.Prefix != "" && q.Prefix != "/") ||
		len(q.Filters) > 0 ||
		len(q.Orders) > 0 ||
		q.Limit > 0 ||
		q.Offset > 0 ||
		!q.KeysOnly {
		// TODO this is overly simplistic, but the only caller is
		// `ipfs refs local` for now, and this gets us moving.
		return nil, errors.New("flatfs only supports listing all keys in random order")
	}

Jeromy's avatar
Jeromy committed
318 319 320 321
	reschan := make(chan query.Result)
	go func() {
		defer close(reschan)
		err := filepath.Walk(fs.path, func(path string, info os.FileInfo, err error) error {
322

Jeromy's avatar
Jeromy committed
323
			if !info.Mode().IsRegular() || strings.HasPrefix(info.Name(), ".") {
Jeromy's avatar
Jeromy committed
324 325
				return nil
			}
326

Jeromy's avatar
Jeromy committed
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
			key, ok := fs.decode(info.Name())
			if !ok {
				log.Warning("failed to decode entry in flatfs")
				return nil
			}

			reschan <- query.Result{
				Entry: query.Entry{
					Key: key.String(),
				},
			}
			return nil
		})
		if err != nil {
			log.Warning("walk failed: ", err)
342
		}
Jeromy's avatar
Jeromy committed
343 344
	}()
	return query.ResultsWithChan(q, reschan), nil
Tommi Virtanen's avatar
Tommi Virtanen committed
345 346
}

Jeromy's avatar
Jeromy committed
347 348 349 350
func (fs *Datastore) Close() error {
	return nil
}

Jeromy's avatar
Jeromy committed
351
type flatfsBatch struct {
Jeromy's avatar
Jeromy committed
352 353 354 355 356 357
	puts    map[datastore.Key]interface{}
	deletes map[datastore.Key]struct{}

	ds *Datastore
}

Jeromy's avatar
Jeromy committed
358
func (fs *Datastore) Batch() (datastore.Batch, error) {
Jeromy's avatar
Jeromy committed
359
	return &flatfsBatch{
Jeromy's avatar
Jeromy committed
360 361 362
		puts:    make(map[datastore.Key]interface{}),
		deletes: make(map[datastore.Key]struct{}),
		ds:      fs,
Jeromy's avatar
Jeromy committed
363
	}, nil
Jeromy's avatar
Jeromy committed
364 365
}

Jeromy's avatar
Jeromy committed
366
func (bt *flatfsBatch) Put(key datastore.Key, val interface{}) error {
Jeromy's avatar
Jeromy committed
367 368 369 370
	bt.puts[key] = val
	return nil
}

Jeromy's avatar
Jeromy committed
371
func (bt *flatfsBatch) Delete(key datastore.Key) error {
Jeromy's avatar
Jeromy committed
372 373 374 375
	bt.deletes[key] = struct{}{}
	return nil
}

Jeromy's avatar
Jeromy committed
376
func (bt *flatfsBatch) Commit() error {
Jeromy's avatar
Jeromy committed
377 378 379 380 381 382 383 384 385 386 387 388 389
	if err := bt.ds.putMany(bt.puts); err != nil {
		return err
	}

	for k, _ := range bt.deletes {
		if err := bt.ds.Delete(k); err != nil {
			return err
		}
	}

	return nil
}

Tommi Virtanen's avatar
Tommi Virtanen committed
390 391 392
var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)

func (*Datastore) IsThreadSafe() {}