convert.go 3.56 KB
Newer Older
1 2 3 4 5 6
// Package flatfs is a Datastore implementation that stores all
// objects in a two-level directory structure in the local file
// system, regardless of the hierarchy of the keys.
package flatfs

import (
7
	"errors"
8 9 10 11
	"fmt"
	"io"
	"os"
	"path/filepath"
12
	"strings"
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31

	"github.com/ipfs/go-datastore"
	"github.com/ipfs/go-datastore/query"
)

func UpgradeV0toV1(path string, prefixLen int) error {
	fun := Prefix(prefixLen)
	err := WriteShardFunc(path, fun)
	if err != nil {
		return err
	}
	err = WriteReadme(path, fun)
	if err != nil {
		return err
	}
	return nil
}

func DowngradeV1toV0(path string) error {
32 33 34 35 36 37 38 39
	fun, err := ReadShardFunc(path)
	if err != nil {
		return err
	} else if fun.funName != "prefix" {
		return fmt.Errorf("%s: can only downgrade datastore that use the 'prefix' sharding function", path)
	}

	err = os.Remove(filepath.Join(path, SHARDING_FN))
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
	if err != nil {
		return err
	}
	err = os.Remove(filepath.Join(path, README_FN))
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	return nil
}

func Move(oldPath string, newPath string, out io.Writer) error {
	oldDS, err := Open(oldPath, false)
	if err != nil {
		return fmt.Errorf("%s: %v", oldPath, err)
	}
55
	oldDS.deactivate()
56 57 58 59
	newDS, err := Open(newPath, false)
	if err != nil {
		return fmt.Errorf("%s: %v", newPath, err)
	}
60
	newDS.deactivate()
61 62 63 64 65 66 67 68 69 70 71

	res, err := oldDS.Query(query.Query{KeysOnly: true})
	if err != nil {
		return err
	}

	if out != nil {
		fmt.Fprintf(out, "Moving Keys...\n")
	}

	// first move the keys
72 73 74 75 76 77 78 79 80 81
	count := 0
	for {
		e, ok := res.NextSync()
		if !ok {
			break
		}
		if e.Error != nil {
			return e.Error
		}

82 83 84 85
		err := moveKey(oldDS, newDS, datastore.RawKey(e.Key))
		if err != nil {
			return err
		}
86 87 88 89 90

		count++
		if out != nil && count%10 == 0 {
			fmt.Fprintf(out, "\r%d keys so far", count)
		}
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
	}

	if out != nil {
		fmt.Fprintf(out, "\nCleaning Up...\n")
	}

	// now walk the old top-level directory
	dir, err := os.Open(oldDS.path)
	if err != nil {
		return err
	}
	defer dir.Close()
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return err
	}
	for _, fn := range names {
		if fn == "." || fn == ".." {
			continue
		}
		oldPath := filepath.Join(oldDS.path, fn)
		inf, err := os.Stat(oldPath)
		if err != nil {
			return err
		}
116 117 118 119 120 121 122
		if inf.IsDir() {
			indir, err := os.Open(oldPath)
			if err != nil {
				return err
			}

			names, err := indir.Readdirnames(-1)
123
			indir.Close()
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
			if err != nil {
				return err
			}

			for _, n := range names {
				p := filepath.Join(oldPath, n)
				// part of unfinished write transaction
				// remove it
				if strings.HasPrefix(n, "put-") {
					err := os.Remove(p)
					if err != nil {
						return err
					}
				} else {
					return errors.New("unknown file in flatfs: " + p)
				}
			}

			err = os.Remove(oldPath)
			if err != nil {
				return err
			}
		} else if fn == SHARDING_FN || fn == README_FN {
			// generated file so just remove it
148 149 150 151 152
			err := os.Remove(oldPath)
			if err != nil {
				return err
			}
		} else {
153
			// else we found something unexpected, so to be safe just move it
Jakub Sztandera's avatar
Jakub Sztandera committed
154
			log.Warningf("found unexpected file in datastore directory: \"%s\", moving anyway\n", fn)
155
			newPath := filepath.Join(newDS.path, fn)
156
			err := os.Rename(oldPath, newPath)
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
			if err != nil {
				return err
			}
		}
	}

	if out != nil {
		fmt.Fprintf(out, "All Done.\n")
	}

	return nil
}

func moveKey(oldDS *Datastore, newDS *Datastore, key datastore.Key) error {
	_, oldPath := oldDS.encode(key)
	dir, newPath := newDS.encode(key)
Steven Allen's avatar
Steven Allen committed
173 174
	err := os.Mkdir(dir, 0755)
	if err != nil && !os.IsExist(err) {
175 176
		return err
	}
177
	err = os.Rename(oldPath, newPath)
178 179 180 181 182
	if err != nil {
		return err
	}
	return nil
}