Enhance Test cases and fix bugs found.

Also sync contents of diskUsage.cache to disk after the initial calculation
and during shutdown.

flatfs.go

@@ -118,8 +118,8 @@ type Datastore struct {
 }
 
 type diskUsageValue struct {
-	diskUsage int64
-	accuracy  initAccuracy
+	DiskUsage int64        `json:"diskUsage"`
+	Accuracy  initAccuracy `json:"accuracy"`
 }
 
 type ShardFunc func(string) string
@@ -763,9 +763,9 @@ func (fs *Datastore) calculateDiskUsage() error {
 		fmt.Println("WARN: re-opening the datastore")
 	}
 
-	fs.storedValue.accuracy = accuracy
+	fs.storedValue.Accuracy = accuracy
 	fs.diskUsage = du
-	fs.writeDiskUsageFile(du)
+	fs.writeDiskUsageFile(du, true)
 
 	return nil
 }
@@ -812,7 +812,7 @@ func (fs *Datastore) checkpointLoop() {
 			du := atomic.LoadInt64(&fs.diskUsage)
 			fs.dirty = true
 			if !more { // shutting down
-				fs.writeDiskUsageFile(du)
+				fs.writeDiskUsageFile(du, true)
 				if fs.dirty {
 					log.Errorf("could not store final value of disk usage to file, future estimates may be inaccurate")
 				}
@@ -823,10 +823,10 @@ func (fs *Datastore) checkpointLoop() {
 			// current one is larger than than `diskUsageCheckpointPercent`
 			// of the checkpointed: store it.
 			newDu := float64(du)
-			lastCheckpointDu := float64(fs.storedValue.diskUsage)
+			lastCheckpointDu := float64(fs.storedValue.DiskUsage)
 			diff := math.Abs(newDu - lastCheckpointDu)
-			if (lastCheckpointDu * diskUsageCheckpointPercent / 100.0) < diff {
-				fs.writeDiskUsageFile(du)
+			if lastCheckpointDu*diskUsageCheckpointPercent < diff*100.0 {
+				fs.writeDiskUsageFile(du, false)
 			}
 			// Otherwise insure the value will be written to disk after
 			// `diskUsageCheckpointTimeout`
@@ -838,25 +838,34 @@ func (fs *Datastore) checkpointLoop() {
 			timerActive = false
 			if fs.dirty {
 				du := atomic.LoadInt64(&fs.diskUsage)
-				fs.writeDiskUsageFile(du)
+				fs.writeDiskUsageFile(du, false)
 			}
 		}
 	}
 }
 
-func (fs *Datastore) writeDiskUsageFile(du int64) {
+func (fs *Datastore) writeDiskUsageFile(du int64, doSync bool) {
 	tmp, err := ioutil.TempFile(fs.path, "du-")
 	if err != nil {
 		log.Warningf("cound not write disk usage: %v", err)
 		return
 	}
 
+	toWrite := fs.storedValue
+	toWrite.DiskUsage = du
 	encoder := json.NewEncoder(tmp)
-	if err := encoder.Encode(&fs.storedValue); err != nil {
+	if err := encoder.Encode(&toWrite); err != nil {
 		log.Warningf("cound not write disk usage: %v", err)
 		return
 	}
 
+	if doSync {
+		if err := tmp.Sync(); err != nil {
+			log.Warningf("cound not sync %s: %v", DiskUsageFile, err)
+			return
+		}
+	}
+
 	if err := tmp.Close(); err != nil {
 		log.Warningf("cound not write disk usage: %v", err)
 		return
@@ -867,7 +876,7 @@ func (fs *Datastore) writeDiskUsageFile(du int64) {
 		return
 	}
 
-	fs.storedValue.diskUsage = du
+	fs.storedValue = toWrite
 	fs.dirty = false
 }
 
@@ -882,7 +891,7 @@ func (fs *Datastore) readDiskUsageFile() int64 {
 	if err != nil {
 		return 0
 	}
-	return fs.storedValue.diskUsage
+	return fs.storedValue.DiskUsage
 }
 
 // DiskUsage implements the PersistentDatastore interface
@@ -907,7 +916,7 @@ func (fs *Datastore) DiskUsage() (uint64, error) {
 // DiskUsage() result, the value returned is implementation defined
 // and for informational purposes only
 func (fs *Datastore) Accuracy() string {
-	return string(fs.storedValue.accuracy)
+	return string(fs.storedValue.Accuracy)
 }
 
 func (fs *Datastore) walk(path string, reschan chan query.Result) error {

flatfs_test.go

@@ -2,6 +2,7 @@ package flatfs_test
 
 import (
 	"encoding/base32"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"math"
@@ -397,7 +398,6 @@ func TestQuerySimple(t *testing.T) { tryAllShardFuncs(t, testQuerySimple) }
 
 func testDiskUsage(dirFunc mkShardFunc, t *testing.T) {
 	temp, cleanup := tempdir(t)
-	t.Log(temp)
 	defer cleanup()
 
 	fs, err := flatfs.CreateOrOpen(temp, dirFunc(2), false)
@@ -445,15 +445,38 @@ func testDiskUsage(dirFunc mkShardFunc, t *testing.T) {
 	}
 	t.Log("duPostDelete:", duDelete)
 
-	// Make sure the accuracy value is correct
-	if fs.Accuracy() != "initial-exact" {
-		t.Errorf("Unexpected value for fs.Accuracy(): %s", fs.Accuracy())
+	du, err := fs.DiskUsage()
+	t.Log("duFinal:", du)
+	if err != nil {
+		t.Fatal(err)
 	}
-
 	fs.Close()
-	os.Remove(filepath.Join(temp, flatfs.DiskUsageFile))
+
+	// Check that disk usage file is correct
+	duB, err := ioutil.ReadFile(filepath.Join(temp, flatfs.DiskUsageFile))
+	if err != nil {
+		t.Fatal(err)
+	}
+	contents := make(map[string]interface{})
+	err = json.Unmarshal(duB, &contents)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure diskUsage value is correct
+	if val, ok := contents["diskUsage"].(float64); !ok || uint64(val) != du {
+		t.Fatalf("Unexpected value for diskUsage in %s: %v (expected %d)",
+			flatfs.DiskUsageFile, contents["diskUsage"], du)
+	}
+
+	// Make sure the accuracy value is correct
+	if val, ok := contents["accuracy"].(string); !ok || val != "initial-exact" {
+		t.Fatalf("Unexpected value for accuracyin %s: %v",
+			flatfs.DiskUsageFile, contents["accuracy"])
+	}
 
 	// Make sure size is correctly calculated on re-open
+	os.Remove(filepath.Join(temp, flatfs.DiskUsageFile))
 	fs, err = flatfs.Open(temp, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
@@ -716,6 +739,19 @@ func testDiskUsageEstimation(dirFunc mkShardFunc, t *testing.T) {
 	if fs.Accuracy() != "initial-approximate" {
 		t.Errorf("Unexpected value for fs.Accuracy(): %s", fs.Accuracy())
 	}
+
+	fs.Close()
+
+	// Reopen into a new variable
+	fs2, err := flatfs.Open(temp, false)
+	if err != nil {
+		t.Fatalf("Open fail: %v\n", err)
+	}
+
+	// Make sure the accuracy value is preserved
+	if fs2.Accuracy() != "initial-approximate" {
+		t.Errorf("Unexpected value for fs.Accuracy(): %s", fs2.Accuracy())
+	}
 }
 
 func TestDiskUsageEstimation(t *testing.T) { tryAllShardFuncs(t, testDiskUsageEstimation) }