Merge pull request #83 from ipfs/feat/fileio

feat: switch to file io and shrink tables

datastore.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	badger "github.com/dgraph-io/badger"
+	options "github.com/dgraph-io/badger/options"
 	ds "github.com/ipfs/go-datastore"
 	dsq "github.com/ipfs/go-datastore/query"
 	logger "github.com/ipfs/go-log"
@@ -71,10 +72,30 @@ func init() {
 		GcDiscardRatio: 0.2,
 		GcInterval:     15 * time.Minute,
 		GcSleep:        10 * time.Second,
-		Options:        badger.DefaultOptions(""),
+		Options:        badger.LSMOnlyOptions(""),
 	}
+	// This is to optimize the database on close so it can be opened
+	// read-only and efficiently queried. We don't do that and hanging on
+	// stop isn't nice.
 	DefaultOptions.Options.CompactL0OnClose = false
+
+	// The alternative is "crash on start and tell the user to fix it". This
+	// will truncate corrupt and unsynced data, which we don't guarantee to
+	// persist anyways.
 	DefaultOptions.Options.Truncate = true
+
+	// Uses less memory, is no slower when writing, and is faster when
+	// reading (in some tests).
+	DefaultOptions.Options.ValueLogLoadingMode = options.FileIO
+
+	// Explicitly set this to mmap. This doesn't use much memory anyways.
+	DefaultOptions.Options.TableLoadingMode = options.MemoryMap
+
+	// Reduce this from 64MiB to 16MiB. That means badger will hold on to
+	// 20MiB by default instead of 80MiB.
+	//
+	// This does not appear to have a significant performance hit.
+	DefaultOptions.Options.MaxTableSize = 16 << 20
 }
 
 var _ ds.Datastore = (*Datastore)(nil)