Merge pull request #584 from jbenet/fix/ipns

fix fuse mounting issues

cmd/ipfs/init.go

@@ -11,6 +11,7 @@ import (
 	core "github.com/jbenet/go-ipfs/core"
 	corecmds "github.com/jbenet/go-ipfs/core/commands"
 	coreunix "github.com/jbenet/go-ipfs/core/coreunix"
+	ipns "github.com/jbenet/go-ipfs/fuse/ipns"
 	ci "github.com/jbenet/go-ipfs/p2p/crypto"
 	peer "github.com/jbenet/go-ipfs/p2p/peer"
 	repo "github.com/jbenet/go-ipfs/repo"
@@ -110,6 +111,11 @@ func doInit(repoRoot string, force bool, nBitsForKeypair int) (interface{}, erro
 		return nil, err
 	}
 
+	err = initializeIpnsKeyspace(repoRoot)
+	if err != nil {
+		return nil, err
+	}
+
 	return nil, nil
 }
 
@@ -138,6 +144,29 @@ func addTheWelcomeFile(repoRoot string) error {
 	return nil
 }
 
+func initializeIpnsKeyspace(repoRoot string) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	r := fsrepo.At(repoRoot)
+	if err := r.Open(); err != nil { // NB: repo is owned by the node
+		return err
+	}
+
+	nd, err := core.NewIPFSNode(ctx, core.Offline(r))
+	if err != nil {
+		return err
+	}
+	defer nd.Close()
+
+	err = nd.SetupOfflineRouting()
+	if err != nil {
+		return err
+	}
+
+	return ipns.InitializeKeyspace(nd, nd.PrivateKey)
+}
+
 func datastoreConfig() (*config.Datastore, error) {
 	dspath, err := config.DataStorePath("")
 	if err != nil {

core/core.go

@@ -34,9 +34,10 @@ import (
 	config "github.com/jbenet/go-ipfs/repo/config"
 	routing "github.com/jbenet/go-ipfs/routing"
 	dht "github.com/jbenet/go-ipfs/routing/dht"
+	offroute "github.com/jbenet/go-ipfs/routing/offline"
+	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
 	util "github.com/jbenet/go-ipfs/util"
 	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
-	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
 	lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables"
 )
 
@@ -350,6 +351,20 @@ func (n *IpfsNode) loadPrivateKey() error {
 
 	n.PrivateKey = sk
 	n.Peerstore.AddPrivKey(n.Identity, n.PrivateKey)
+	n.Peerstore.AddPubKey(n.Identity, sk.GetPublic())
+	return nil
+}
+
+// SetupOfflineRouting loads the local nodes private key and
+// uses it to instantiate a routing system in offline mode.
+// This is primarily used for offline ipns modifications.
+func (n *IpfsNode) SetupOfflineRouting() error {
+	err := n.loadPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	n.Routing = offroute.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
 	return nil
 }
 
@@ -421,6 +436,6 @@ func constructPeerHost(ctx context.Context, cfg *config.Config, id peer.ID, ps p
 
 func constructDHTRouting(ctx context.Context, host p2phost.Host, ds datastore.ThreadSafeDatastore) (*dht.IpfsDHT, error) {
 	dhtRouting := dht.NewDHT(ctx, host, ds)
-	dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord
+	dhtRouting.Validator[IpnsValidatorTag] = namesys.ValidateIpnsRecord
 	return dhtRouting, nil
 }

fuse/ipns/ipns_unix.go

@@ -16,6 +16,7 @@ import (
 	core "github.com/jbenet/go-ipfs/core"
 	chunk "github.com/jbenet/go-ipfs/importer/chunk"
 	mdag "github.com/jbenet/go-ipfs/merkledag"
+	nsys "github.com/jbenet/go-ipfs/namesys"
 	ci "github.com/jbenet/go-ipfs/p2p/crypto"
 	ft "github.com/jbenet/go-ipfs/unixfs"
 	uio "github.com/jbenet/go-ipfs/unixfs/io"
@@ -32,6 +33,24 @@ var (
 	longRepublishTimeout  = time.Millisecond * 500
 )
 
+// InitializeKeyspace sets the ipns record for the given key to
+// point to an empty directory.
+func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
+	emptyDir := &mdag.Node{Data: ft.FolderPBData()}
+	k, err := n.DAG.Add(emptyDir)
+	if err != nil {
+		return err
+	}
+
+	pub := nsys.NewRoutingPublisher(n.Routing)
+	err = pub.Publish(key, k.B58String())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // FileSystem is the readwrite IPNS Fuse Filesystem.
 type FileSystem struct {
 	Ipfs     *core.IpfsNode

routing/dht/dht.go

@@ -10,12 +10,14 @@ import (
 	"sync"
 	"time"
 
+	ci "github.com/jbenet/go-ipfs/p2p/crypto"
 	host "github.com/jbenet/go-ipfs/p2p/host"
 	peer "github.com/jbenet/go-ipfs/p2p/peer"
 	protocol "github.com/jbenet/go-ipfs/p2p/protocol"
 	routing "github.com/jbenet/go-ipfs/routing"
 	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
 	kb "github.com/jbenet/go-ipfs/routing/kbucket"
+	record "github.com/jbenet/go-ipfs/routing/record"
 	"github.com/jbenet/go-ipfs/thirdparty/eventlog"
 	u "github.com/jbenet/go-ipfs/util"
 
@@ -52,8 +54,7 @@ type IpfsDHT struct {
 	birth    time.Time  // When this peer started up
 	diaglock sync.Mutex // lock to make diagnostics work better
 
-	// record validator funcs
-	Validators map[string]ValidatorFunc
+	Validator record.Validator // record validator funcs
 
 	ctxgroup.ContextGroup
 }
@@ -79,8 +80,8 @@ func NewDHT(ctx context.Context, h host.Host, dstore ds.ThreadSafeDatastore) *Ip
 	dht.routingTable = kb.NewRoutingTable(20, kb.ConvertPeerID(dht.self), time.Minute, dht.peerstore)
 	dht.birth = time.Now()
 
-	dht.Validators = make(map[string]ValidatorFunc)
-	dht.Validators["pk"] = ValidatePublicKeyRecord
+	dht.Validator = make(record.Validator)
+	dht.Validator["pk"] = record.ValidatePublicKeyRecord
 
 	if doPinging {
 		dht.Children().Add(1)
@@ -234,9 +235,25 @@ func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) {
 	return rec.GetValue(), nil
 }
 
+// getOwnPrivateKey attempts to load the local peers private
+// key from the peerstore.
+func (dht *IpfsDHT) getOwnPrivateKey() (ci.PrivKey, error) {
+	sk := dht.peerstore.PrivKey(dht.self)
+	if sk == nil {
+		log.Errorf("%s dht cannot get own private key!", dht.self)
+		return nil, fmt.Errorf("cannot get private key to sign record!")
+	}
+	return sk, nil
+}
+
 // putLocal stores the key value pair in the datastore
 func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error {
-	rec, err := dht.makePutRecord(key, value)
+	sk, err := dht.getOwnPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	rec, err := record.MakePutRecord(sk, key, value)
 	if err != nil {
 		return err
 	}

routing/dht/dht_test.go

@@ -38,7 +38,7 @@ func setupDHT(ctx context.Context, t *testing.T) *IpfsDHT {
 	dss := dssync.MutexWrap(ds.NewMapDatastore())
 	d := NewDHT(ctx, h, dss)
 
-	d.Validators["v"] = func(u.Key, []byte) error {
+	d.Validator["v"] = func(u.Key, []byte) error {
 		return nil
 	}
 	return d
@@ -142,8 +142,8 @@ func TestValueGetSet(t *testing.T) {
 	vf := func(u.Key, []byte) error {
 		return nil
 	}
-	dhtA.Validators["v"] = vf
-	dhtB.Validators["v"] = vf
+	dhtA.Validator["v"] = vf
+	dhtB.Validator["v"] = vf
 
 	connect(t, ctx, dhtA, dhtB)
 

routing/dht/ext_test.go

@@ -11,6 +11,7 @@ import (
 	peer "github.com/jbenet/go-ipfs/p2p/peer"
 	routing "github.com/jbenet/go-ipfs/routing"
 	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
+	record "github.com/jbenet/go-ipfs/routing/record"
 	u "github.com/jbenet/go-ipfs/util"
 
 	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
@@ -98,7 +99,13 @@ func TestGetFailures(t *testing.T) {
 	{
 		typ := pb.Message_GET_VALUE
 		str := "hello"
-		rec, err := d.makePutRecord(u.Key(str), []byte("blah"))
+
+		sk, err := d.getOwnPrivateKey()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		rec, err := record.MakePutRecord(sk, u.Key(str), []byte("blah"))
 		if err != nil {
 			t.Fatal(err)
 		}

routing/dht/records.go

 package dht
 
 import (
-	"bytes"
-	"errors"
 	"fmt"
-	"strings"
 
 	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
-	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
 
 	ci "github.com/jbenet/go-ipfs/p2p/crypto"
-	"github.com/jbenet/go-ipfs/p2p/peer"
+	peer "github.com/jbenet/go-ipfs/p2p/peer"
 	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
 	u "github.com/jbenet/go-ipfs/util"
 	ctxutil "github.com/jbenet/go-ipfs/util/ctx"
 )
 
-// ValidatorFunc is a function that is called to validate a given
-// type of DHTRecord.
-type ValidatorFunc func(u.Key, []byte) error
-
-// ErrBadRecord is returned any time a dht record is found to be
-// incorrectly formatted or signed.
-var ErrBadRecord = errors.New("bad dht record")
-
-// ErrInvalidRecordType is returned if a DHTRecord keys prefix
-// is not found in the Validator map of the DHT.
-var ErrInvalidRecordType = errors.New("invalid record keytype")
-
 // KeyForPublicKey returns the key used to retrieve public keys
 // from the dht.
 func KeyForPublicKey(id peer.ID) u.Key {
 	return u.Key("/pk/" + string(id))
 }
 
-// RecordBlobForSig returns the blob protected by the record signature
-func RecordBlobForSig(r *pb.Record) []byte {
-	k := []byte(r.GetKey())
-	v := []byte(r.GetValue())
-	a := []byte(r.GetAuthor())
-	return bytes.Join([][]byte{k, v, a}, []byte{})
-}
-
-// creates and signs a dht record for the given key/value pair
-func (dht *IpfsDHT) makePutRecord(key u.Key, value []byte) (*pb.Record, error) {
-	record := new(pb.Record)
-
-	record.Key = proto.String(string(key))
-	record.Value = value
-	record.Author = proto.String(string(dht.self))
-	blob := RecordBlobForSig(record)
-
-	sk := dht.peerstore.PrivKey(dht.self)
-	if sk == nil {
-		log.Errorf("%s dht cannot get own private key!", dht.self)
-		return nil, fmt.Errorf("cannot get private key to sign record!")
-	}
-
-	sig, err := sk.Sign(blob)
-	if err != nil {
-		return nil, err
-	}
-
-	record.Signature = sig
-	return record, nil
-}
-
 func (dht *IpfsDHT) getPublicKeyOnline(ctx context.Context, p peer.ID) (ci.PubKey, error) {
 	log.Debugf("getPublicKey for: %s", p)
 
@@ -155,7 +107,7 @@ func (dht *IpfsDHT) verifyRecordLocally(r *pb.Record) error {
 		return fmt.Errorf("do not have public key for %s", p)
 	}
 
-	return dht.verifyRecord(r, pk)
+	return dht.Validator.VerifyRecord(r, pk)
 }
 
 // verifyRecordOnline verifies a record, searching the DHT for the public key
@@ -172,50 +124,5 @@ func (dht *IpfsDHT) verifyRecordOnline(ctx context.Context, r *pb.Record) error
 		return err
 	}
 
-	return dht.verifyRecord(r, pk)
-}
-
-func (dht *IpfsDHT) verifyRecord(r *pb.Record, pk ci.PubKey) error {
-	// First, validate the signature
-	blob := RecordBlobForSig(r)
-	ok, err := pk.Verify(blob, r.GetSignature())
-	if err != nil {
-		log.Error("Signature verify failed.")
-		return err
-	}
-	if !ok {
-		log.Error("dht found a forged record! (ignored)")
-		return ErrBadRecord
-	}
-
-	// Now, check validity func
-	parts := strings.Split(r.GetKey(), "/")
-	if len(parts) < 3 {
-		log.Infof("Record key does not have validator: %s", u.Key(r.GetKey()))
-		return nil
-	}
-
-	fnc, ok := dht.Validators[parts[1]]
-	if !ok {
-		log.Errorf("Unrecognized key prefix: %s", parts[1])
-		return ErrInvalidRecordType
-	}
-
-	return fnc(u.Key(r.GetKey()), r.GetValue())
-}
-
-// ValidatePublicKeyRecord implements ValidatorFunc and
-// verifies that the passed in record value is the PublicKey
-// that matches the passed in key.
-func ValidatePublicKeyRecord(k u.Key, val []byte) error {
-	keyparts := bytes.Split([]byte(k), []byte("/"))
-	if len(keyparts) < 3 {
-		return errors.New("invalid key")
-	}
-
-	pkh := u.Hash(val)
-	if !bytes.Equal(keyparts[2], pkh) {
-		return errors.New("public key does not match storage key")
-	}
-	return nil
+	return dht.Validator.VerifyRecord(r, pk)
 }

routing/dht/routing.go

@@ -12,6 +12,7 @@ import (
 	"github.com/jbenet/go-ipfs/routing"
 	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
 	kb "github.com/jbenet/go-ipfs/routing/kbucket"
+	record "github.com/jbenet/go-ipfs/routing/record"
 	u "github.com/jbenet/go-ipfs/util"
 	errors "github.com/jbenet/go-ipfs/util/debugerror"
 	pset "github.com/jbenet/go-ipfs/util/peerset"
@@ -36,7 +37,12 @@ func (dht *IpfsDHT) PutValue(ctx context.Context, key u.Key, value []byte) error
 		return err
 	}
 
-	rec, err := dht.makePutRecord(key, value)
+	sk, err := dht.getOwnPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	rec, err := record.MakePutRecord(sk, key, value)
 	if err != nil {
 		log.Error("Creation of record failed!")
 		return err
@@ -75,6 +81,8 @@ func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
 	if err == nil {
 		log.Debug("have it locally")
 		return val, nil
+	} else {
+		log.Debug("failed to get value locally: %s", err)
 	}
 
 	// get closest peers in the routing table

routing/offline/offline.go

+package offline
+
+import (
+	"errors"
+	"time"
+
+	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
+	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+	ci "github.com/jbenet/go-ipfs/p2p/crypto"
+	"github.com/jbenet/go-ipfs/p2p/peer"
+	routing "github.com/jbenet/go-ipfs/routing"
+	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
+	record "github.com/jbenet/go-ipfs/routing/record"
+	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
+	u "github.com/jbenet/go-ipfs/util"
+)
+
+var log = eventlog.Logger("offlinerouting")
+
+var ErrOffline = errors.New("routing system in offline mode")
+
+func NewOfflineRouter(dstore ds.Datastore, privkey ci.PrivKey) routing.IpfsRouting {
+	return &offlineRouting{
+		datastore: dstore,
+		sk:        privkey,
+	}
+}
+
+// offlineRouting implements the IpfsRouting interface,
+// but only provides the capability to Put and Get signed dht
+// records to and from the local datastore.
+type offlineRouting struct {
+	datastore ds.Datastore
+	sk        ci.PrivKey
+}
+
+func (c *offlineRouting) PutValue(ctx context.Context, key u.Key, val []byte) error {
+	rec, err := record.MakePutRecord(c.sk, key, val)
+	if err != nil {
+		return err
+	}
+	data, err := proto.Marshal(rec)
+	if err != nil {
+		return err
+	}
+
+	return c.datastore.Put(key.DsKey(), data)
+}
+
+func (c *offlineRouting) GetValue(ctx context.Context, key u.Key) ([]byte, error) {
+	v, err := c.datastore.Get(key.DsKey())
+	if err != nil {
+		return nil, err
+	}
+
+	byt, ok := v.([]byte)
+	if !ok {
+		return nil, errors.New("value stored in datastore not []byte")
+	}
+	rec := new(pb.Record)
+	err = proto.Unmarshal(byt, rec)
+	if err != nil {
+		return nil, err
+	}
+
+	return rec.GetValue(), nil
+}
+
+func (c *offlineRouting) FindProviders(ctx context.Context, key u.Key) ([]peer.PeerInfo, error) {
+	return nil, ErrOffline
+}
+
+func (c *offlineRouting) FindPeer(ctx context.Context, pid peer.ID) (peer.PeerInfo, error) {
+	return peer.PeerInfo{}, ErrOffline
+}
+
+func (c *offlineRouting) FindProvidersAsync(ctx context.Context, k u.Key, max int) <-chan peer.PeerInfo {
+	out := make(chan peer.PeerInfo)
+	close(out)
+	return out
+}
+
+func (c *offlineRouting) Provide(_ context.Context, key u.Key) error {
+	return ErrOffline
+}
+
+func (c *offlineRouting) Ping(ctx context.Context, p peer.ID) (time.Duration, error) {
+	return 0, ErrOffline
+}
+
+// ensure offlineRouting matches the IpfsRouting interface
+var _ routing.IpfsRouting = &offlineRouting{}

routing/record/record.go

+package record
+
+import (
+	"bytes"
+
+	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
+
+	ci "github.com/jbenet/go-ipfs/p2p/crypto"
+	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
+	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
+	u "github.com/jbenet/go-ipfs/util"
+)
+
+var log = eventlog.Logger("routing/record")
+
+// MakePutRecord creates and signs a dht record for the given key/value pair
+func MakePutRecord(sk ci.PrivKey, key u.Key, value []byte) (*pb.Record, error) {
+	record := new(pb.Record)
+
+	record.Key = proto.String(string(key))
+	record.Value = value
+
+	pkh, err := sk.GetPublic().Hash()
+	if err != nil {
+		return nil, err
+	}
+
+	record.Author = proto.String(string(pkh))
+	blob := RecordBlobForSig(record)
+
+	sig, err := sk.Sign(blob)
+	if err != nil {
+		return nil, err
+	}
+
+	record.Signature = sig
+	return record, nil
+}
+
+// RecordBlobForSig returns the blob protected by the record signature
+func RecordBlobForSig(r *pb.Record) []byte {
+	k := []byte(r.GetKey())
+	v := []byte(r.GetValue())
+	a := []byte(r.GetAuthor())
+	return bytes.Join([][]byte{k, v, a}, []byte{})
+}

routing/record/validation.go

+package record
+
+import (
+	"bytes"
+	"errors"
+	"strings"
+
+	ci "github.com/jbenet/go-ipfs/p2p/crypto"
+	pb "github.com/jbenet/go-ipfs/routing/dht/pb"
+	u "github.com/jbenet/go-ipfs/util"
+)
+
+// ValidatorFunc is a function that is called to validate a given
+// type of DHTRecord.
+type ValidatorFunc func(u.Key, []byte) error
+
+// ErrBadRecord is returned any time a dht record is found to be
+// incorrectly formatted or signed.
+var ErrBadRecord = errors.New("bad dht record")
+
+// ErrInvalidRecordType is returned if a DHTRecord keys prefix
+// is not found in the Validator map of the DHT.
+var ErrInvalidRecordType = errors.New("invalid record keytype")
+
+// Validator is an object that helps ensure routing records are valid.
+// It is a collection of validator functions, each of which implements
+// its own notion of validity.
+type Validator map[string]ValidatorFunc
+
+// VerifyRecord checks a record and ensures it is still valid.
+// It runs needed validators
+func (v Validator) VerifyRecord(r *pb.Record, pk ci.PubKey) error {
+	// First, validate the signature
+	blob := RecordBlobForSig(r)
+	ok, err := pk.Verify(blob, r.GetSignature())
+	if err != nil {
+		log.Error("Signature verify failed.")
+		return err
+	}
+	if !ok {
+		log.Error("dht found a forged record! (ignored)")
+		return ErrBadRecord
+	}
+
+	// Now, check validity func
+	parts := strings.Split(r.GetKey(), "/")
+	if len(parts) < 3 {
+		log.Infof("Record key does not have validator: %s", u.Key(r.GetKey()))
+		return nil
+	}
+
+	fnc, ok := v[parts[1]]
+	if !ok {
+		log.Errorf("Unrecognized key prefix: %s", parts[1])
+		return ErrInvalidRecordType
+	}
+
+	return fnc(u.Key(r.GetKey()), r.GetValue())
+}
+
+// ValidatePublicKeyRecord implements ValidatorFunc and
+// verifies that the passed in record value is the PublicKey
+// that matches the passed in key.
+func ValidatePublicKeyRecord(k u.Key, val []byte) error {
+	keyparts := bytes.Split([]byte(k), []byte("/"))
+	if len(keyparts) < 3 {
+		return errors.New("invalid key")
+	}
+
+	pkh := u.Hash(val)
+	if !bytes.Equal(keyparts[2], pkh) {
+		return errors.New("public key does not match storage key")
+	}
+	return nil
+}