core.go 26.2 KB
Newer Older
1 2 3 4
/*
Package core implements the IpfsNode object and related methods.

Packages underneath core/ provide a (relatively) stable, low-level API
5 6 7 8
to carry out most IPFS-related tasks.  For more details on the other
interfaces and how core/... fits into the bigger IPFS picture, see:

  $ godoc github.com/ipfs/go-ipfs
9
*/
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
10 11
package core

Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
12
import (
Jakub Sztandera's avatar
Jakub Sztandera committed
13
	"bytes"
14
	"context"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
15
	"errors"
Juan Batiz-Benet's avatar
go fmt  
Juan Batiz-Benet committed
16
	"fmt"
17
	"io"
18
	"io/ioutil"
Jeromy's avatar
Jeromy committed
19
	"net"
20 21
	"os"
	"strings"
22
	"time"
23

24 25 26 27
	bserv "github.com/ipfs/go-ipfs/blockservice"
	bitswap "github.com/ipfs/go-ipfs/exchange/bitswap"
	bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
	rp "github.com/ipfs/go-ipfs/exchange/reprovide"
28
	filestore "github.com/ipfs/go-ipfs/filestore"
29 30 31 32 33
	mount "github.com/ipfs/go-ipfs/fuse/mount"
	merkledag "github.com/ipfs/go-ipfs/merkledag"
	mfs "github.com/ipfs/go-ipfs/mfs"
	namesys "github.com/ipfs/go-ipfs/namesys"
	ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher"
Łukasz Magiera's avatar
Łukasz Magiera committed
34
	p2p "github.com/ipfs/go-ipfs/p2p"
35
	"github.com/ipfs/go-ipfs/path/resolver"
36 37 38 39
	pin "github.com/ipfs/go-ipfs/pin"
	repo "github.com/ipfs/go-ipfs/repo"
	config "github.com/ipfs/go-ipfs/repo/config"
	ft "github.com/ipfs/go-ipfs/unixfs"
Jeromy's avatar
Jeromy committed
40

Steven Allen's avatar
Steven Allen committed
41
	addrutil "gx/ipfs/QmNSWW3Sb4eju4o2djPQ1L1c2Zj9XN9sMYJL8r1cbxdc6b/go-addr-util"
Steven Allen's avatar
Steven Allen committed
42
	yamux "gx/ipfs/QmNWCEvi7bPRcvqAV8AKLGVNoQdArWi7NJayka2SM4XtRe/go-smux-yamux"
Steven Allen's avatar
Steven Allen committed
43 44 45 46 47
	discovery "gx/ipfs/QmNh1kGFFdsPu79KNSaL4NUKUPb4Eiz4KHdMtFY6664RDp/go-libp2p/p2p/discovery"
	p2pbhost "gx/ipfs/QmNh1kGFFdsPu79KNSaL4NUKUPb4Eiz4KHdMtFY6664RDp/go-libp2p/p2p/host/basic"
	rhost "gx/ipfs/QmNh1kGFFdsPu79KNSaL4NUKUPb4Eiz4KHdMtFY6664RDp/go-libp2p/p2p/host/routed"
	identify "gx/ipfs/QmNh1kGFFdsPu79KNSaL4NUKUPb4Eiz4KHdMtFY6664RDp/go-libp2p/p2p/protocol/identify"
	ping "gx/ipfs/QmNh1kGFFdsPu79KNSaL4NUKUPb4Eiz4KHdMtFY6664RDp/go-libp2p/p2p/protocol/ping"
Steven Allen's avatar
Steven Allen committed
48
	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
Steven Allen's avatar
Steven Allen committed
49 50
	p2phost "gx/ipfs/QmNmJZL7FQySMtE2BQuLMuZg2EB2CLEunJJUSVSc9YnnbV/go-libp2p-host"
	logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log"
51
	goprocess "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess"
Steven Allen's avatar
Steven Allen committed
52
	floodsub "gx/ipfs/QmSFihvoND3eDaAYRCeLgLPt62yCPgMZs1NSZmKFEtJQQw/go-libp2p-floodsub"
Jeromy's avatar
Jeromy committed
53
	mamask "gx/ipfs/QmSMZwvs3n4GBikZ7hKzT17c3bk65FmyZo2JqtJ16swqCv/multiaddr-filter"
Steven Allen's avatar
Steven Allen committed
54 55 56
	swarm "gx/ipfs/QmSwZMWwFZSUpe5muU2xgTUwppH24KfMwdPXiwbEp2c6G5/go-libp2p-swarm"
	routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing"
	circuit "gx/ipfs/QmVTnHzuyECV9JzbXXfZRj1pKtgknp1esamUb2EH33mJkA/go-libp2p-circuit"
Steven Allen's avatar
Steven Allen committed
57
	mssmux "gx/ipfs/QmVniQJkdzLZaZwzwMdd3dJTvWiJ1DQEkreVy6hs6h7Vk5/go-smux-multistream"
Steven Allen's avatar
Steven Allen committed
58
	ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr"
Hector Sanjuan's avatar
Hector Sanjuan committed
59
	ds "gx/ipfs/QmXRKBQA4wXP7xWbFiZsR1GP4HV6wMDQ1aWFxZZ4uBcPX9/go-datastore"
Steven Allen's avatar
Steven Allen committed
60
	pstore "gx/ipfs/QmXauCuJzmzapetmC6W4TuDJLL1yFFrVzSHoWv8YdbmnxH/go-libp2p-peerstore"
61 62 63
	nilrouting "gx/ipfs/QmXtoXbu9ReyV6Q4kDQ5CF9wXQNDY1PdHc4HhfxRR5AHB3/go-ipfs-routing/none"
	offroute "gx/ipfs/QmXtoXbu9ReyV6Q4kDQ5CF9wXQNDY1PdHc4HhfxRR5AHB3/go-ipfs-routing/offline"
	dht "gx/ipfs/QmY1y2M1aCcVhy8UuTbZJBvuFbegZm47f9cDAdgxiehQfx/go-libp2p-kad-dht"
Steven Allen's avatar
Steven Allen committed
64
	smux "gx/ipfs/QmY9JXR3FupnYAYJWK9aMr9bCpqWKcToQ1tz8DVGTrHpHw/go-stream-muxer"
Steven Allen's avatar
Steven Allen committed
65 66 67
	connmgr "gx/ipfs/QmZ1R2LxRZTUaeuMFEtQigzHfFCv3hLYBi5316aZ7YUeyf/go-libp2p-connmgr"
	ipnet "gx/ipfs/QmZPrWxuM8GHr4cGKbyF5CCT11sFUP9hgqpeUHALvx2nUr/go-libp2p-interface-pnet"
	peer "gx/ipfs/QmZoWKhxUmZ2seW4BzX6fJkNR8hh9PsGModr7q171yq2SS/go-libp2p-peer"
Hector Sanjuan's avatar
Hector Sanjuan committed
68
	bstore "gx/ipfs/QmaG4DZ4JaqEfvPWt5nPPgoTzhc1tr1T3f4Nu9Jpdm8ymY/go-ipfs-blockstore"
69
	ic "gx/ipfs/QmaPbCnUMBohSGo3KnxEa2bHqyJVVeEEcwtqJAYxerieBo/go-libp2p-crypto"
Steven Allen's avatar
Steven Allen committed
70 71
	ifconnmgr "gx/ipfs/Qmax8X1Kfahf5WfSB68EWDG3d3qyS3Sqs1v412fjPTfRwx/go-libp2p-interface-connmgr"
	mplex "gx/ipfs/Qmc14vuKyGqX27RvBhekYytxSFJpaEgQVuVJgKSm69MEix/go-smux-multiplex"
Steven Allen's avatar
Steven Allen committed
72
	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
73
	exchange "gx/ipfs/QmdcAXgEHUueP4A7b5hjabKn2EooeHgMreMvFC249dGCgc/go-ipfs-exchange-interface"
Steven Allen's avatar
Steven Allen committed
74
	metrics "gx/ipfs/QmdeBtQGXjSt7cb97nx9JyLHHv5va2LyEAue7Q5tDFzpLy/go-libp2p-metrics"
75
	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
Steven Allen's avatar
Steven Allen committed
76
	pnet "gx/ipfs/QmenK8PgcpM2KYzEKnGx1LyN1QXawswM2F6HktCbWKuC1b/go-libp2p-pnet"
Steven Allen's avatar
Steven Allen committed
77
	mafilter "gx/ipfs/Qmf2UAmRwDG4TvnkQpHZWPAzw7rpCYVhxmRXmYxXr5LD1g/go-maddr-filter"
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
78 79
)

Jeromy's avatar
Jeromy committed
80
const IpnsValidatorTag = "ipns"
81

82
const kReprovideFrequency = time.Hour * 12
83
const discoveryConnTimeout = time.Second * 30
Jeromy's avatar
Jeromy committed
84

Jeromy's avatar
Jeromy committed
85
var log = logging.Logger("core")
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
86

87 88 89 90
type mode int

const (
	// zero value is not a valid mode, must be explicitly set
91
	localMode mode = iota
92 93 94 95
	offlineMode
	onlineMode
)

96 97 98 99
func init() {
	identify.ClientVersion = "go-ipfs/" + config.CurrentVersionNumber + "/" + config.CurrentCommit
}

Juan Batiz-Benet's avatar
go lint  
Juan Batiz-Benet committed
100
// IpfsNode is IPFS Core module. It represents an IPFS instance.
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
101 102
type IpfsNode struct {

103
	// Self
104
	Identity peer.ID // the local node's identity
105

106
	Repo repo.Repo
107 108

	// Local node
109 110 111 112
	Pinning         pin.Pinner // the pinning manager
	Mounts          Mounts     // current mount state, if any.
	PrivateKey      ic.PrivKey // the local node's private Key
	PNetFingerprint []byte     // fingerprint of private network
113 114

	// Services
Jeromy's avatar
Jeromy committed
115
	Peerstore  pstore.Peerstore     // storage for other Peer instances
116
	Blockstore bstore.GCBlockstore  // the block store (lower level)
117 118 119
	Filestore  *filestore.Filestore // the filestore blockstore
	BaseBlocks bstore.Blockstore    // the raw blockstore, no filestore wrapping
	GCLocker   bstore.GCLocker      // the locker used to protect the blockstore during gc
120
	Blocks     bserv.BlockService   // the block service, get/add blocks.
121
	DAG        ipld.DAGService      // the merkle dag service, get/add objects.
122
	Resolver   *resolver.Resolver   // the path resolution system
Jeromy's avatar
Jeromy committed
123
	Reporter   metrics.Reporter
124
	Discovery  discovery.Service
Jeromy's avatar
Jeromy committed
125
	FilesRoot  *mfs.Root
126 127

	// Online
128 129 130 131 132
	PeerHost     p2phost.Host        // the network host (server+client)
	Bootstrapper io.Closer           // the periodic bootstrapper
	Routing      routing.IpfsRouting // the routing system. recommend ipfs-dht
	Exchange     exchange.Interface  // the block exchange + strategy (bitswap)
	Namesys      namesys.NameSystem  // the name system, resolves paths to hashes
Jeromy's avatar
Jeromy committed
133 134
	Ping         *ping.PingService
	Reprovider   *rp.Reprovider // the value reprovider system
Jeromy's avatar
Jeromy committed
135
	IpnsRepub    *ipnsrp.Republisher
136

Jeromy's avatar
Jeromy committed
137
	Floodsub *floodsub.PubSub
Łukasz Magiera's avatar
Łukasz Magiera committed
138
	P2P      *p2p.P2P
Jeromy's avatar
Jeromy committed
139

140 141
	proc goprocess.Process
	ctx  context.Context
142

Jeromy's avatar
Jeromy committed
143
	mode         mode
144
	localModeSet bool
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
145 146
}

147 148 149 150 151 152 153 154
// Mounts defines what the node's mount state is. This should
// perhaps be moved to the daemon or mount. It's here because
// it needs to be accessible across daemon requests.
type Mounts struct {
	Ipfs mount.Mount
	Ipns mount.Mount
}

vyzo's avatar
vyzo committed
155
func (n *IpfsNode) startOnlineServices(ctx context.Context, routingOption RoutingOption, hostOption HostOption, do DiscoveryOption, pubsub, ipnsps, mplex bool) error {
156 157

	if n.PeerHost != nil { // already online.
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
158
		return errors.New("node already online")
159 160 161
	}

	// load private key
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
162
	if err := n.LoadPrivateKey(); err != nil {
163 164 165
		return err
	}

Jeromy's avatar
Jeromy committed
166
	// get undialable addrs from config
167 168 169 170
	cfg, err := n.Repo.Config()
	if err != nil {
		return err
	}
Jeromy's avatar
Jeromy committed
171
	var addrfilter []*net.IPNet
172
	for _, s := range cfg.Swarm.AddrFilters {
Jeromy's avatar
Jeromy committed
173 174
		f, err := mamask.NewMask(s)
		if err != nil {
175
			return fmt.Errorf("incorrectly formatted address filter in config: %s", s)
Jeromy's avatar
Jeromy committed
176 177 178 179
		}
		addrfilter = append(addrfilter, f)
	}

180 181 182 183 184
	if !cfg.Swarm.DisableBandwidthMetrics {
		// Set reporter
		n.Reporter = metrics.NewBandwidthCounter()
	}

185 186
	tpt := makeSmuxTransport(mplex)

Jakub Sztandera's avatar
Jakub Sztandera committed
187 188 189 190 191 192 193 194 195
	swarmkey, err := n.Repo.SwarmKey()
	if err != nil {
		return err
	}

	var protec ipnet.Protector
	if swarmkey != nil {
		protec, err = pnet.NewProtector(bytes.NewReader(swarmkey))
		if err != nil {
196
			return fmt.Errorf("failed to configure private network: %s", err)
Jakub Sztandera's avatar
Jakub Sztandera committed
197
		}
198
		n.PNetFingerprint = protec.Fingerprint()
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
		go func() {
			t := time.NewTicker(30 * time.Second)
			<-t.C // swallow one tick
			for {
				select {
				case <-t.C:
					if ph := n.PeerHost; ph != nil {
						if len(ph.Network().Peers()) == 0 {
							log.Warning("We are in private network and have no peers.")
							log.Warning("This might be configuration mistake.")
						}
					}
				case <-n.Process().Closing():
					t.Stop()
					return
				}
			}
		}()
Jakub Sztandera's avatar
Jakub Sztandera committed
217 218
	}

219 220 221 222 223
	addrsFactory, err := makeAddrsFactory(cfg.Addresses)
	if err != nil {
		return err
	}

Jeromy's avatar
Jeromy committed
224 225 226 227 228
	connmgr, err := constructConnMgr(cfg.Swarm.ConnMgr)
	if err != nil {
		return err
	}

229 230 231
	hostopts := &ConstructPeerHostOpts{
		AddrsFactory:      addrsFactory,
		DisableNatPortMap: cfg.Swarm.DisableNatPortMap,
vyzo's avatar
vyzo committed
232 233
		DisableRelay:      cfg.Swarm.DisableRelay,
		EnableRelayHop:    cfg.Swarm.EnableRelayHop,
Jeromy's avatar
Jeromy committed
234
		ConnectionManager: connmgr,
235
	}
Jakub Sztandera's avatar
Jakub Sztandera committed
236
	peerhost, err := hostOption(ctx, n.Identity, n.Peerstore, n.Reporter,
237
		addrfilter, tpt, protec, hostopts)
vyzo's avatar
vyzo committed
238

239
	if err != nil {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
240
		return err
241 242
	}

243
	if err := n.startOnlineServicesWithHost(ctx, peerhost, routingOption); err != nil {
244
		return err
245 246 247
	}

	// Ok, now we're ready to listen.
Łukasz Magiera's avatar
Łukasz Magiera committed
248
	if err := startListening(n.PeerHost, cfg); err != nil {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
249
		return err
250
	}
251

vyzo's avatar
vyzo committed
252
	if pubsub || ipnsps {
253 254 255 256 257
		service, err := floodsub.NewFloodSub(ctx, peerhost)
		if err != nil {
			return err
		}
		n.Floodsub = service
Jeromy's avatar
Jeromy committed
258
	}
Jeromy's avatar
Jeromy committed
259

vyzo's avatar
vyzo committed
260 261 262 263 264 265 266
	if ipnsps {
		err = namesys.AddPubsubNameSystem(ctx, n.Namesys, n.PeerHost, n.Routing, n.Repo.Datastore(), n.Floodsub)
		if err != nil {
			return err
		}
	}

Łukasz Magiera's avatar
Łukasz Magiera committed
267
	n.P2P = p2p.NewP2P(n.Identity, n.PeerHost, n.Peerstore)
268

269
	// setup local discovery
Jeromy's avatar
Jeromy committed
270
	if do != nil {
Jeromy's avatar
Jeromy committed
271
		service, err := do(ctx, n.PeerHost)
Jeromy's avatar
Jeromy committed
272
		if err != nil {
Jeromy's avatar
Jeromy committed
273 274 275 276
			log.Error("mdns error: ", err)
		} else {
			service.RegisterNotifee(n)
			n.Discovery = service
Jeromy's avatar
Jeromy committed
277
		}
278 279
	}

280
	return n.Bootstrap(DefaultBootstrapConfig)
281 282
}

Jeromy's avatar
Jeromy committed
283 284
func constructConnMgr(cfg config.ConnMgr) (ifconnmgr.ConnManager, error) {
	switch cfg.Type {
285 286 287 288
	case "":
		// 'default' value is the basic connection manager
		return connmgr.NewConnManager(config.DefaultConnMgrLowWater, config.DefaultConnMgrHighWater, config.DefaultConnMgrGracePeriod), nil
	case "none":
Jeromy's avatar
Jeromy committed
289 290 291 292 293 294 295 296 297 298 299 300 301
		return nil, nil
	case "basic":
		grace, err := time.ParseDuration(cfg.GracePeriod)
		if err != nil {
			return nil, fmt.Errorf("parsing Swarm.ConnMgr.GracePeriod: %s", err)
		}

		return connmgr.NewConnManager(cfg.LowWater, cfg.HighWater, grace), nil
	default:
		return nil, fmt.Errorf("unrecognized ConnMgr.Type: %q", cfg.Type)
	}
}

Łukasz Magiera's avatar
Łukasz Magiera committed
302 303 304 305 306 307
func (n *IpfsNode) startLateOnlineServices(ctx context.Context) error {
	cfg, err := n.Repo.Config()
	if err != nil {
		return err
	}

308
	var keyProvider rp.KeyChanFunc
Łukasz Magiera's avatar
Łukasz Magiera committed
309 310 311 312 313 314 315 316 317 318 319

	switch cfg.Reprovider.Strategy {
	case "all":
		fallthrough
	case "":
		keyProvider = rp.NewBlockstoreProvider(n.Blockstore)
	case "roots":
		keyProvider = rp.NewPinnedProvider(n.Pinning, n.DAG, true)
	case "pinned":
		keyProvider = rp.NewPinnedProvider(n.Pinning, n.DAG, false)
	default:
320
		return fmt.Errorf("unknown reprovider strategy '%s'", cfg.Reprovider.Strategy)
Łukasz Magiera's avatar
Łukasz Magiera committed
321
	}
322
	n.Reprovider = rp.NewReprovider(ctx, n.Routing, keyProvider)
Łukasz Magiera's avatar
Łukasz Magiera committed
323

324 325 326 327 328
	reproviderInterval := kReprovideFrequency
	if cfg.Reprovider.Interval != "" {
		dur, err := time.ParseDuration(cfg.Reprovider.Interval)
		if err != nil {
			return err
Łukasz Magiera's avatar
Łukasz Magiera committed
329 330
		}

331
		reproviderInterval = dur
Łukasz Magiera's avatar
Łukasz Magiera committed
332 333
	}

334 335
	go n.Reprovider.Run(reproviderInterval)

Łukasz Magiera's avatar
Łukasz Magiera committed
336 337 338
	return nil
}

339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
func makeAddrsFactory(cfg config.Addresses) (p2pbhost.AddrsFactory, error) {
	var annAddrs []ma.Multiaddr
	for _, addr := range cfg.Announce {
		maddr, err := ma.NewMultiaddr(addr)
		if err != nil {
			return nil, err
		}
		annAddrs = append(annAddrs, maddr)
	}

	filters := mafilter.NewFilters()
	noAnnAddrs := map[string]bool{}
	for _, addr := range cfg.NoAnnounce {
		f, err := mamask.NewMask(addr)
		if err == nil {
			filters.AddDialFilter(f)
			continue
		}
		maddr, err := ma.NewMultiaddr(addr)
		if err != nil {
			return nil, err
		}
		noAnnAddrs[maddr.String()] = true
	}

	return func(allAddrs []ma.Multiaddr) []ma.Multiaddr {
		var addrs []ma.Multiaddr
		if len(annAddrs) > 0 {
			addrs = annAddrs
		} else {
			addrs = allAddrs
		}

		var out []ma.Multiaddr
		for _, maddr := range addrs {
			// check for exact matches
			ok, _ := noAnnAddrs[maddr.String()]
			// check for /ipcidr matches
			if !ok && !filters.AddrBlocked(maddr) {
				out = append(out, maddr)
			}
		}
		return out
	}, nil
}

385 386 387 388
func makeSmuxTransport(mplexExp bool) smux.Transport {
	mstpt := mssmux.NewBlankTransport()

	ymxtpt := &yamux.Transport{
Steven Allen's avatar
Steven Allen committed
389
		AcceptBacklog:          512,
390 391 392 393 394 395 396
		ConnectionWriteTimeout: time.Second * 10,
		KeepAliveInterval:      time.Second * 30,
		EnableKeepAlive:        true,
		MaxStreamWindowSize:    uint32(1024 * 512),
		LogOutput:              ioutil.Discard,
	}

397 398 399 400
	if os.Getenv("YAMUX_DEBUG") != "" {
		ymxtpt.LogOutput = os.Stderr
	}

401 402 403 404 405 406 407 408 409 410 411 412 413 414
	mstpt.AddTransport("/yamux/1.0.0", ymxtpt)

	if mplexExp {
		mstpt.AddTransport("/mplex/6.7.0", mplex.DefaultTransport)
	}

	// Allow muxer preference order overriding
	if prefs := os.Getenv("LIBP2P_MUX_PREFS"); prefs != "" {
		mstpt.OrderPreference = strings.Fields(prefs)
	}

	return mstpt
}

Jeromy's avatar
Jeromy committed
415 416
func setupDiscoveryOption(d config.Discovery) DiscoveryOption {
	if d.MDNS.Enabled {
Jeromy's avatar
Jeromy committed
417
		return func(ctx context.Context, h p2phost.Host) (discovery.Service, error) {
Jeromy's avatar
Jeromy committed
418 419 420
			if d.MDNS.Interval == 0 {
				d.MDNS.Interval = 5
			}
Jeromy's avatar
Jeromy committed
421
			return discovery.NewMdnsService(ctx, h, time.Duration(d.MDNS.Interval)*time.Second, discovery.ServiceTag)
Jeromy's avatar
Jeromy committed
422 423 424 425 426
		}
	}
	return nil
}

427 428
// HandlePeerFound attempts to connect to peer from `PeerInfo`, if it fails
// logs a warning log.
Jeromy's avatar
Jeromy committed
429
func (n *IpfsNode) HandlePeerFound(p pstore.PeerInfo) {
430
	log.Warning("trying peer info: ", p)
431
	ctx, cancel := context.WithTimeout(n.Context(), discoveryConnTimeout)
rht's avatar
rht committed
432
	defer cancel()
433
	if err := n.PeerHost.Connect(ctx, p); err != nil {
434 435 436 437
		log.Warning("Failed to connect to peer found by discovery: ", err)
	}
}

438 439
// startOnlineServicesWithHost  is the set of services which need to be
// initialized with the host and _before_ we start listening.
440
func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost.Host, routingOption RoutingOption) error {
441
	// setup diagnostics service
Jeromy's avatar
Jeromy committed
442
	n.Ping = ping.NewPingService(host)
443 444

	// setup routing service
445
	r, err := routingOption(ctx, host, n.Repo.Datastore())
Jeromy's avatar
Jeromy committed
446
	if err != nil {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
447
		return err
448
	}
Jeromy's avatar
Jeromy committed
449
	n.Routing = r
450

451 452 453
	// Wrap standard peer host with routing system to allow unknown peer lookups
	n.PeerHost = rhost.Wrap(host, n.Routing)

454
	// setup exchange service
455
	bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing)
Łukasz Magiera's avatar
Łukasz Magiera committed
456
	n.Exchange = bitswap.New(ctx, bitswapNetwork, n.Blockstore)
457

458 459 460 461 462
	size, err := n.getCacheSize()
	if err != nil {
		return err
	}

463
	// setup name system
464
	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size)
465

Jeromy's avatar
Jeromy committed
466
	// setup ipns republishing
467
	return n.setupIpnsRepublisher()
468 469
}

470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
// getCacheSize returns cache life and cache size
func (n *IpfsNode) getCacheSize() (int, error) {
	cfg, err := n.Repo.Config()
	if err != nil {
		return 0, err
	}

	cs := cfg.Ipns.ResolveCacheSize
	if cs == 0 {
		cs = 128
	}
	if cs < 0 {
		return 0, fmt.Errorf("cannot specify negative resolve cache size")
	}
	return cs, nil
}

487
func (n *IpfsNode) setupIpnsRepublisher() error {
Jeromy's avatar
Jeromy committed
488 489 490 491
	cfg, err := n.Repo.Config()
	if err != nil {
		return err
	}
492

493
	n.IpnsRepub = ipnsrp.NewRepublisher(n.Routing, n.Repo.Datastore(), n.PrivateKey, n.Repo.Keystore())
494

Jeromy's avatar
Jeromy committed
495 496 497 498 499 500
	if cfg.Ipns.RepublishPeriod != "" {
		d, err := time.ParseDuration(cfg.Ipns.RepublishPeriod)
		if err != nil {
			return fmt.Errorf("failure to parse config setting IPNS.RepublishPeriod: %s", err)
		}

501
		if !u.Debug && (d < time.Minute || d > (time.Hour*24)) {
Jeromy's avatar
Jeromy committed
502 503 504 505 506 507
			return fmt.Errorf("config setting IPNS.RepublishPeriod is not between 1min and 1day: %s", d)
		}

		n.IpnsRepub.Interval = d
	}

508 509 510 511 512 513 514 515 516
	if cfg.Ipns.RecordLifetime != "" {
		d, err := time.ParseDuration(cfg.Ipns.RepublishPeriod)
		if err != nil {
			return fmt.Errorf("failure to parse config setting IPNS.RecordLifetime: %s", err)
		}

		n.IpnsRepub.RecordLifetime = d
	}

Jeromy's avatar
Jeromy committed
517 518
	n.Process().Go(n.IpnsRepub.Run)

519 520 521
	return nil
}

522 523 524 525 526 527 528 529 530 531 532 533
// Process returns the Process object
func (n *IpfsNode) Process() goprocess.Process {
	return n.proc
}

// Close calls Close() on the Process object
func (n *IpfsNode) Close() error {
	return n.proc.Close()
}

// Context returns the IpfsNode context
func (n *IpfsNode) Context() context.Context {
534 535 536
	if n.ctx == nil {
		n.ctx = context.TODO()
	}
537 538 539
	return n.ctx
}

540 541
// teardown closes owned children. If any errors occur, this function returns
// the first error.
Brian Tiger Chow's avatar
Brian Tiger Chow committed
542
func (n *IpfsNode) teardown() error {
543
	log.Debug("core is shutting down...")
544 545
	// owned objects are closed in this teardown to ensure that they're closed
	// regardless of which constructor was used to add them to the node.
Jeromy's avatar
Jeromy committed
546 547
	var closers []io.Closer

548
	// NOTE: The order that objects are added(closed) matters, if an object
Jeromy's avatar
Jeromy committed
549 550 551 552 553
	// needs to use another during its shutdown/cleanup process, it should be
	// closed before that other object

	if n.FilesRoot != nil {
		closers = append(closers, n.FilesRoot)
Jeromy's avatar
Jeromy committed
554
	}
555

556 557 558 559
	if n.Exchange != nil {
		closers = append(closers, n.Exchange)
	}

560
	if n.Mounts.Ipfs != nil && !n.Mounts.Ipfs.IsActive() {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
561 562
		closers = append(closers, mount.Closer(n.Mounts.Ipfs))
	}
563
	if n.Mounts.Ipns != nil && !n.Mounts.Ipns.IsActive() {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
564 565 566
		closers = append(closers, mount.Closer(n.Mounts.Ipns))
	}

Jeromy's avatar
Jeromy committed
567 568 569 570
	if dht, ok := n.Routing.(*dht.IpfsDHT); ok {
		closers = append(closers, dht.Process())
	}

Jeromy's avatar
Jeromy committed
571 572 573 574
	if n.Blocks != nil {
		closers = append(closers, n.Blocks)
	}

Jeromy's avatar
Jeromy committed
575 576
	if n.Bootstrapper != nil {
		closers = append(closers, n.Bootstrapper)
577 578
	}

Jeromy's avatar
Jeromy committed
579 580
	if n.PeerHost != nil {
		closers = append(closers, n.PeerHost)
581
	}
582

Jeromy's avatar
Jeromy committed
583 584 585
	// Repo closed last, most things need to preserve state here
	closers = append(closers, n.Repo)

586
	var errs []error
587
	for _, closer := range closers {
588 589
		if err := closer.Close(); err != nil {
			errs = append(errs, err)
590 591 592 593
		}
	}
	if len(errs) > 0 {
		return errs[0]
Brian Tiger Chow's avatar
Brian Tiger Chow committed
594 595
	}
	return nil
Brian Tiger Chow's avatar
Brian Tiger Chow committed
596 597
}

598
// OnlineMode returns whether or not the IpfsNode is in OnlineMode.
Brian Tiger Chow's avatar
Brian Tiger Chow committed
599
func (n *IpfsNode) OnlineMode() bool {
600 601 602 603 604 605
	switch n.mode {
	case onlineMode:
		return true
	default:
		return false
	}
Brian Tiger Chow's avatar
Brian Tiger Chow committed
606 607
}

608
// SetLocal will set the IpfsNode to local mode
609 610 611 612 613 614 615
func (n *IpfsNode) SetLocal(isLocal bool) {
	if isLocal {
		n.mode = localMode
	}
	n.localModeSet = true
}

616
// LocalMode returns whether or not the IpfsNode is in LocalMode
617 618 619 620 621 622 623 624 625 626 627 628 629
func (n *IpfsNode) LocalMode() bool {
	if !n.localModeSet {
		// programmer error should not happen
		panic("local mode not set")
	}
	switch n.mode {
	case localMode:
		return true
	default:
		return false
	}
}

630
// Bootstrap will set and call the IpfsNodes bootstrap function.
631
func (n *IpfsNode) Bootstrap(cfg BootstrapConfig) error {
632 633

	// TODO what should return value be when in offlineMode?
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
634 635 636 637
	if n.Routing == nil {
		return nil
	}

638 639 640 641 642 643 644
	if n.Bootstrapper != nil {
		n.Bootstrapper.Close() // stop previous bootstrap process.
	}

	// if the caller did not specify a bootstrap peer function, get the
	// freshest bootstrap peers from config. this responds to live changes.
	if cfg.BootstrapPeers == nil {
Jeromy's avatar
Jeromy committed
645
		cfg.BootstrapPeers = func() []pstore.PeerInfo {
646
			ps, err := n.loadBootstrapPeers()
647
			if err != nil {
648
				log.Warning("failed to parse bootstrap peers from config")
649 650 651 652 653 654 655 656 657
				return nil
			}
			return ps
		}
	}

	var err error
	n.Bootstrapper, err = Bootstrap(n, cfg)
	return err
658 659
}

660 661
func (n *IpfsNode) loadID() error {
	if n.Identity != "" {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
662
		return errors.New("identity already loaded")
663 664
	}

665 666 667 668 669 670
	cfg, err := n.Repo.Config()
	if err != nil {
		return err
	}

	cid := cfg.Identity.PeerID
671
	if cid == "" {
672
		return errors.New("identity was not set in config (was 'ipfs init' run?)")
673 674
	}
	if len(cid) == 0 {
675
		return errors.New("no peer ID in config! (was 'ipfs init' run?)")
676 677
	}

Steven Allen's avatar
Steven Allen committed
678 679 680 681 682 683
	id, err := peer.IDB58Decode(cid)
	if err != nil {
		return fmt.Errorf("peer ID invalid: %s", err)
	}

	n.Identity = id
684 685
	return nil
}
686

687
// GetKey will return a key from the Keystore with name `name`.
688 689 690 691 692 693 694 695
func (n *IpfsNode) GetKey(name string) (ic.PrivKey, error) {
	if name == "self" {
		return n.PrivateKey, nil
	} else {
		return n.Repo.Keystore().Get(name)
	}
}

Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
696
func (n *IpfsNode) LoadPrivateKey() error {
697
	if n.Identity == "" || n.Peerstore == nil {
Łukasz Magiera's avatar
Łukasz Magiera committed
698
		return errors.New("loaded private key out of order")
699 700
	}

701
	if n.PrivateKey != nil {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
702
		return errors.New("private key already loaded")
703 704
	}

705 706 707 708 709 710
	cfg, err := n.Repo.Config()
	if err != nil {
		return err
	}

	sk, err := loadPrivateKey(&cfg.Identity, n.Identity)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
711
	if err != nil {
712
		return err
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
713
	}
714

715 716
	n.PrivateKey = sk
	n.Peerstore.AddPrivKey(n.Identity, n.PrivateKey)
Jeromy's avatar
Jeromy committed
717 718 719 720
	n.Peerstore.AddPubKey(n.Identity, sk.GetPublic())
	return nil
}

Jeromy's avatar
Jeromy committed
721
func (n *IpfsNode) loadBootstrapPeers() ([]pstore.PeerInfo, error) {
722 723 724 725 726 727
	cfg, err := n.Repo.Config()
	if err != nil {
		return nil, err
	}

	parsed, err := cfg.BootstrapPeers()
728 729 730 731 732 733
	if err != nil {
		return nil, err
	}
	return toPeerInfos(parsed), nil
}

Jeromy's avatar
Jeromy committed
734
func (n *IpfsNode) loadFilesRoot() error {
Jeromy's avatar
Jeromy committed
735
	dsk := ds.NewKey("/local/filesroot")
Jeromy's avatar
Jeromy committed
736 737
	pf := func(ctx context.Context, c *cid.Cid) error {
		return n.Repo.Datastore().Put(dsk, c.Bytes())
Jeromy's avatar
Jeromy committed
738 739
	}

740
	var nd *merkledag.ProtoNode
Jeromy's avatar
Jeromy committed
741 742 743 744
	val, err := n.Repo.Datastore().Get(dsk)

	switch {
	case err == ds.ErrNotFound || val == nil:
745
		nd = ft.EmptyDirNode()
746
		err := n.DAG.Add(n.Context(), nd)
Jeromy's avatar
Jeromy committed
747 748 749 750
		if err != nil {
			return fmt.Errorf("failure writing to dagstore: %s", err)
		}
	case err == nil:
Jeromy's avatar
Jeromy committed
751 752 753 754 755
		c, err := cid.Cast(val.([]byte))
		if err != nil {
			return err
		}

756
		rnd, err := n.DAG.Get(n.Context(), c)
Jeromy's avatar
Jeromy committed
757 758 759
		if err != nil {
			return fmt.Errorf("error loading filesroot from DAG: %s", err)
		}
760 761 762 763 764 765 766

		pbnd, ok := rnd.(*merkledag.ProtoNode)
		if !ok {
			return merkledag.ErrNotProtobuf
		}

		nd = pbnd
Jeromy's avatar
Jeromy committed
767 768 769 770 771 772 773 774 775 776 777 778 779
	default:
		return err
	}

	mr, err := mfs.NewRoot(n.Context(), n.DAG, nd, pf)
	if err != nil {
		return err
	}

	n.FilesRoot = mr
	return nil
}

Jeromy's avatar
Jeromy committed
780 781 782
// SetupOfflineRouting loads the local nodes private key and
// uses it to instantiate a routing system in offline mode.
// This is primarily used for offline ipns modifications.
Jeromy's avatar
Jeromy committed
783
func (n *IpfsNode) SetupOfflineRouting() error {
784 785 786 787
	if n.Routing != nil {
		// Routing was already set up
		return nil
	}
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
788
	err := n.LoadPrivateKey()
Jeromy's avatar
Jeromy committed
789 790 791 792 793
	if err != nil {
		return err
	}

	n.Routing = offroute.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
794

795 796 797 798 799 800
	size, err := n.getCacheSize()
	if err != nil {
		return err
	}

	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size)
801

802
	return nil
803 804 805 806
}

func loadPrivateKey(cfg *config.Identity, id peer.ID) (ic.PrivKey, error) {
	sk, err := cfg.DecodePrivateKey("passphrase todo!")
807 808 809
	if err != nil {
		return nil, err
	}
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
810

811 812 813 814
	id2, err := peer.IDFromPrivateKey(sk)
	if err != nil {
		return nil, err
	}
815

816 817
	if id2 != id {
		return nil, fmt.Errorf("private key in config does not match id: %s != %s", id, id2)
818 819
	}

820
	return sk, nil
821
}
822

823
func listenAddresses(cfg *config.Config) ([]ma.Multiaddr, error) {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
824 825 826
	var listen []ma.Multiaddr
	for _, addr := range cfg.Addresses.Swarm {
		maddr, err := ma.NewMultiaddr(addr)
827
		if err != nil {
Łukasz Magiera's avatar
Łukasz Magiera committed
828
			return nil, fmt.Errorf("failure to parse config.Addresses.Swarm: %s", cfg.Addresses.Swarm)
829
		}
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
830
		listen = append(listen, maddr)
831 832 833 834
	}

	return listen, nil
}
835

Kevin Atkinson's avatar
Kevin Atkinson committed
836
type ConstructPeerHostOpts struct {
837
	AddrsFactory      p2pbhost.AddrsFactory
vyzo's avatar
vyzo committed
838 839 840
	DisableNatPortMap bool
	DisableRelay      bool
	EnableRelayHop    bool
Jeromy's avatar
Jeromy committed
841
	ConnectionManager ifconnmgr.ConnManager
Kevin Atkinson's avatar
Kevin Atkinson committed
842 843 844
}

type HostOption func(ctx context.Context, id peer.ID, ps pstore.Peerstore, bwr metrics.Reporter, fs []*net.IPNet, tpt smux.Transport, protc ipnet.Protector, opts *ConstructPeerHostOpts) (p2phost.Host, error)
Jeromy's avatar
Jeromy committed
845 846 847

var DefaultHostOption HostOption = constructPeerHost

848
// isolates the complex initialization steps
Kevin Atkinson's avatar
Kevin Atkinson committed
849
func constructPeerHost(ctx context.Context, id peer.ID, ps pstore.Peerstore, bwr metrics.Reporter, fs []*net.IPNet, tpt smux.Transport, protec ipnet.Protector, opts *ConstructPeerHostOpts) (p2phost.Host, error) {
850 851

	// no addresses to begin with. we'll start later.
Jakub Sztandera's avatar
Jakub Sztandera committed
852
	swrm, err := swarm.NewSwarmWithProtector(ctx, nil, id, ps, protec, tpt, bwr)
853
	if err != nil {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
854
		return nil, err
855
	}
856

857 858
	network := (*swarm.Network)(swrm)

859 860 861 862
	for _, f := range fs {
		network.Swarm().Filters.AddDialFilter(f)
	}

Kevin Atkinson's avatar
Kevin Atkinson committed
863 864 865 866
	hostOpts := []interface{}{bwr}
	if !opts.DisableNatPortMap {
		hostOpts = append(hostOpts, p2pbhost.NATPortMap)
	}
Jeromy's avatar
Jeromy committed
867 868 869
	if opts.ConnectionManager != nil {
		hostOpts = append(hostOpts, opts.ConnectionManager)
	}
Kevin Atkinson's avatar
Kevin Atkinson committed
870

871
	addrsFactory := opts.AddrsFactory
vyzo's avatar
vyzo committed
872
	if !opts.DisableRelay {
873
		if addrsFactory != nil {
874
			addrsFactory = composeAddrsFactory(addrsFactory, filterRelayAddrs)
875
		} else {
876
			addrsFactory = filterRelayAddrs
877 878 879 880 881
		}
	}

	if addrsFactory != nil {
		hostOpts = append(hostOpts, addrsFactory)
vyzo's avatar
vyzo committed
882 883
	}

Kevin Atkinson's avatar
Kevin Atkinson committed
884
	host := p2pbhost.New(network, hostOpts...)
Jeromy's avatar
Jeromy committed
885

vyzo's avatar
vyzo committed
886 887 888 889 890 891 892 893
	if !opts.DisableRelay {
		var relayOpts []circuit.RelayOpt
		if opts.EnableRelayHop {
			relayOpts = append(relayOpts, circuit.OptHop)
		}

		err := circuit.AddRelayTransport(ctx, host, relayOpts...)
		if err != nil {
894
			host.Close()
vyzo's avatar
vyzo committed
895 896 897 898
			return nil, err
		}
	}

899 900 901
	return host, nil
}

902 903 904 905 906 907 908 909 910 911 912 913
func filterRelayAddrs(addrs []ma.Multiaddr) []ma.Multiaddr {
	var raddrs []ma.Multiaddr
	for _, addr := range addrs {
		_, err := addr.ValueForProtocol(circuit.P_CIRCUIT)
		if err == nil {
			continue
		}
		raddrs = append(raddrs, addr)
	}
	return raddrs
}

914 915 916 917 918 919
func composeAddrsFactory(f, g p2pbhost.AddrsFactory) p2pbhost.AddrsFactory {
	return func(addrs []ma.Multiaddr) []ma.Multiaddr {
		return f(g(addrs))
	}
}

920
// startListening on the network addresses
Łukasz Magiera's avatar
Łukasz Magiera committed
921
func startListening(host p2phost.Host, cfg *config.Config) error {
922 923
	listenAddrs, err := listenAddresses(cfg)
	if err != nil {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
924
		return err
925 926
	}

927
	// make sure we error out if our config does not have addresses we can use
928
	log.Debugf("Config.Addresses.Swarm:%s", listenAddrs)
929
	filteredAddrs := addrutil.FilterUsableAddrs(listenAddrs)
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
930
	log.Debugf("Config.Addresses.Swarm:%s (filtered)", filteredAddrs)
931
	if len(filteredAddrs) < 1 {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
932
		return fmt.Errorf("addresses in config not usable: %s", listenAddrs)
933 934
	}

935 936 937
	// Actually start listening:
	if err := host.Network().Listen(filteredAddrs...); err != nil {
		return err
938 939
	}

940
	// list out our addresses
941
	addrs, err := host.Network().InterfaceListenAddresses()
942
	if err != nil {
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
943
		return err
944
	}
Juan Batiz-Benet's avatar
Juan Batiz-Benet committed
945
	log.Infof("Swarm listening at: %s", addrs)
946
	return nil
947
}
948

949
func constructDHTRouting(ctx context.Context, host p2phost.Host, dstore ds.Batching) (routing.IpfsRouting, error) {
950
	dhtRouting := dht.NewDHT(ctx, host, dstore)
951
	dhtRouting.Validator[IpnsValidatorTag] = namesys.NewIpnsRecordValidator(host.Peerstore())
952
	dhtRouting.Selector[IpnsValidatorTag] = namesys.IpnsSelectorFunc
953 954
	return dhtRouting, nil
}
Jeromy's avatar
Jeromy committed
955

956
func constructClientDHTRouting(ctx context.Context, host p2phost.Host, dstore ds.Batching) (routing.IpfsRouting, error) {
Jeromy's avatar
Jeromy committed
957
	dhtRouting := dht.NewDHTClient(ctx, host, dstore)
958
	dhtRouting.Validator[IpnsValidatorTag] = namesys.NewIpnsRecordValidator(host.Peerstore())
Jeromy's avatar
Jeromy committed
959 960 961 962
	dhtRouting.Selector[IpnsValidatorTag] = namesys.IpnsSelectorFunc
	return dhtRouting, nil
}

963
type RoutingOption func(context.Context, p2phost.Host, ds.Batching) (routing.IpfsRouting, error)
Jeromy's avatar
Jeromy committed
964

Jeromy's avatar
Jeromy committed
965
type DiscoveryOption func(context.Context, p2phost.Host) (discovery.Service, error)
Jeromy's avatar
Jeromy committed
966

967
var DHTOption RoutingOption = constructDHTRouting
Jeromy's avatar
Jeromy committed
968
var DHTClientOption RoutingOption = constructClientDHTRouting
969
var NilRouterOption RoutingOption = nilrouting.ConstructNilRouting