Merge branch 'master' into feat/namestream

License: MIT
Signed-off-by: Łukasz Magiera <magik6k@gmail.com>

coreapi.go

@@ -34,6 +34,12 @@ type CoreAPI interface {
 	// Dht returns an implementation of Dht API
 	Dht() DhtAPI
 
+	// Swarm returns an implementation of Swarm API
+	Swarm() SwarmAPI
+
+	// PubSub returns an implementation of PubSub API
+	PubSub() PubSubAPI
+
 	// ResolvePath resolves the path using Unixfs resolver
 	ResolvePath(context.Context, Path) (ResolvedPath, error)
 

dht.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
+	pstore "gx/ipfs/QmXEyLwySuDMXejWBu8XwdkX2WuGKk8x9jFwz8js7j72UX/go-libp2p-peerstore"
 	peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer"
-	pstore "gx/ipfs/QmfAQMFpgDU2U4BXG64qVr8HSiictfWvkSBz7Y2oDj65st/go-libp2p-peerstore"
 )
 
 // DhtAPI specifies the interface to the DHT

errors.go

@@ -4,5 +4,5 @@ import "errors"
 
 var (
 	ErrIsDir   = errors.New("object is a directory")
-	ErrOffline = errors.New("can't resolve, ipfs node is offline")
+	ErrOffline = errors.New("this action must be run in online mode, try running 'ipfs daemon' first")
 )

key.go

@@ -33,6 +33,9 @@ type KeyAPI interface {
 	// List lists keys stored in keystore
 	List(ctx context.Context) ([]Key, error)
 
+	// Self returns the 'main' node key
+	Self(ctx context.Context) (Key, error)
+
 	// Remove removes keys from keystore. Returns ipns path of the removed key
 	Remove(ctx context.Context, name string) (Key, error)
 }

name.go

@@ -3,9 +3,13 @@ package iface
 import (
 	"context"
 
+	"errors"
+
 	options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 )
 
+var ErrResolveFailed = errors.New("could not resolve name")
+
 // IpnsEntry specifies the interface to IpnsEntries
 type IpnsEntry interface {
 	// Name returns IpnsEntry name
@@ -14,6 +18,11 @@ type IpnsEntry interface {
 	Value() Path
 }
 
+type IpnsResult struct {
+	Path
+	Err error
+}
+
 // NameAPI specifies the interface to IPNS.
 //
 // IPNS is a PKI namespace, where names are the hashes of public keys, and the
@@ -28,4 +37,11 @@ type NameAPI interface {
 
 	// Resolve attempts to resolve the newest version of the specified name
 	Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (Path, error)
+
+	// Search is a version of Resolve which outputs paths as they are discovered,
+	// reducing the time to first entry
+	//
+	// Note that by default only the last path returned before the channel closes
+	// can be considered 'safe'.
+	Search(ctx context.Context, name string, opts ...options.NameResolveOption) (<-chan IpnsResult, error)
 }

options/name.go

@@ -13,6 +13,10 @@ const (
 type NamePublishSettings struct {
 	ValidTime time.Duration
 	Key       string
+
+	TTL *time.Duration
+
+	AllowOffline bool
 }
 
 type NameResolveSettings struct {
@@ -29,6 +33,8 @@ func NamePublishOptions(opts ...NamePublishOption) (*NamePublishSettings, error)
 	options := &NamePublishSettings{
 		ValidTime: DefaultNameValidTime,
 		Key:       "self",
+
+		AllowOffline: false,
 	}
 
 	for _, opt := range opts {
@@ -82,6 +88,24 @@ func (nameOpts) Key(key string) NamePublishOption {
 	}
 }
 
+// AllowOffline is an option for Name.Publish which specifies whether to allow
+// publishing when the node is offline. Default value is false
+func (nameOpts) AllowOffline(allow bool) NamePublishOption {
+	return func(settings *NamePublishSettings) error {
+		settings.AllowOffline = allow
+		return nil
+	}
+}
+
+// TTL is an option for Name.Publish which specifies the time duration the
+// published record should be cached for (caution: experimental).
+func (nameOpts) TTL(ttl time.Duration) NamePublishOption {
+	return func(settings *NamePublishSettings) error {
+		settings.TTL = &ttl
+		return nil
+	}
+}
+
 // Local is an option for Name.Resolve which specifies if the lookup should be
 // offline. Default value is false
 func (nameOpts) Local(local bool) NameResolveOption {

options/pubsub.go

+package options
+
+type PubSubPeersSettings struct {
+	Topic string
+}
+
+type PubSubSubscribeSettings struct {
+	Discover bool
+}
+
+type PubSubPeersOption func(*PubSubPeersSettings) error
+type PubSubSubscribeOption func(*PubSubSubscribeSettings) error
+
+func PubSubPeersOptions(opts ...PubSubPeersOption) (*PubSubPeersSettings, error) {
+	options := &PubSubPeersSettings{
+		Topic: "",
+	}
+
+	for _, opt := range opts {
+		err := opt(options)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return options, nil
+}
+
+func PubSubSubscribeOptions(opts ...PubSubSubscribeOption) (*PubSubSubscribeSettings, error) {
+	options := &PubSubSubscribeSettings{
+		Discover: false,
+	}
+
+	for _, opt := range opts {
+		err := opt(options)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return options, nil
+}
+
+type pubsubOpts struct{}
+
+var PubSub pubsubOpts
+
+func (pubsubOpts) Topic(topic string) PubSubPeersOption {
+	return func(settings *PubSubPeersSettings) error {
+		settings.Topic = topic
+		return nil
+	}
+}
+
+func (pubsubOpts) Discover(discover bool) PubSubSubscribeOption {
+	return func(settings *PubSubSubscribeSettings) error {
+		settings.Discover = discover
+		return nil
+	}
+}

options/unixfs.go

+package options
+
+import (
+	"errors"
+	"fmt"
+
+	cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
+	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
+	dag "gx/ipfs/QmXTw4By9FMZAt7qJm4JoJuNBrBgqMMzkS4AjKc4zqTUVd/go-merkledag"
+)
+
+type Layout int
+
+const (
+	BalancedLayout Layout = iota
+	TrickleLayout
+)
+
+type UnixfsAddSettings struct {
+	CidVersion int
+	MhType     uint64
+
+	Inline       bool
+	InlineLimit  int
+	RawLeaves    bool
+	RawLeavesSet bool
+
+	Chunker string
+	Layout  Layout
+
+	Pin      bool
+	OnlyHash bool
+	Local    bool
+	FsCache  bool
+	NoCopy   bool
+
+	Wrap      bool
+	Hidden    bool
+	StdinName string
+
+	Events   chan<- interface{}
+	Silent   bool
+	Progress bool
+}
+
+type UnixfsAddOption func(*UnixfsAddSettings) error
+
+func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, error) {
+	options := &UnixfsAddSettings{
+		CidVersion: -1,
+		MhType:     mh.SHA2_256,
+
+		Inline:       false,
+		InlineLimit:  32,
+		RawLeaves:    false,
+		RawLeavesSet: false,
+
+		Chunker: "size-262144",
+		Layout:  BalancedLayout,
+
+		Pin:      false,
+		OnlyHash: false,
+		Local:    false,
+		FsCache:  false,
+		NoCopy:   false,
+
+		Wrap:      false,
+		Hidden:    false,
+		StdinName: "",
+
+		Events:   nil,
+		Silent:   false,
+		Progress: false,
+	}
+
+	for _, opt := range opts {
+		err := opt(options)
+		if err != nil {
+			return nil, cid.Prefix{}, err
+		}
+	}
+
+	// nocopy -> rawblocks
+	if options.NoCopy && !options.RawLeaves {
+		// fixed?
+		if options.RawLeavesSet {
+			return nil, cid.Prefix{}, fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well")
+		}
+
+		// No, satisfy mandatory constraint.
+		options.RawLeaves = true
+	}
+
+	// (hash != "sha2-256") -> CIDv1
+	if options.MhType != mh.SHA2_256 {
+		switch options.CidVersion {
+		case 0:
+			return nil, cid.Prefix{}, errors.New("CIDv0 only supports sha2-256")
+		case 1, -1:
+			options.CidVersion = 1
+		default:
+			return nil, cid.Prefix{}, fmt.Errorf("unknown CID version: %d", options.CidVersion)
+		}
+	} else {
+		if options.CidVersion < 0 {
+			// Default to CIDv0
+			options.CidVersion = 0
+		}
+	}
+
+	// cidV1 -> raw blocks (by default)
+	if options.CidVersion > 0 && !options.RawLeavesSet {
+		options.RawLeaves = true
+	}
+
+	prefix, err := dag.PrefixForCidVersion(options.CidVersion)
+	if err != nil {
+		return nil, cid.Prefix{}, err
+	}
+
+	prefix.MhType = options.MhType
+	prefix.MhLength = -1
+
+	return options, prefix, nil
+}
+
+type unixfsOpts struct{}
+
+var Unixfs unixfsOpts
+
+// CidVersion specifies which CID version to use. Defaults to 0 unless an option
+// that depends on CIDv1 is passed.
+func (unixfsOpts) CidVersion(version int) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.CidVersion = version
+		return nil
+	}
+}
+
+// Hash function to use. Implies CIDv1 if not set to sha2-256 (default).
+//
+// Table of functions is declared in https://github.com/multiformats/go-multihash/blob/master/multihash.go
+func (unixfsOpts) Hash(mhtype uint64) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.MhType = mhtype
+		return nil
+	}
+}
+
+// RawLeaves specifies whether to use raw blocks for leaves (data nodes with no
+// links) instead of wrapping them with unixfs structures.
+func (unixfsOpts) RawLeaves(enable bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.RawLeaves = enable
+		settings.RawLeavesSet = true
+		return nil
+	}
+}
+
+// Inline tells the adder to inline small blocks into CIDs
+func (unixfsOpts) Inline(enable bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Inline = enable
+		return nil
+	}
+}
+
+// InlineLimit sets the amount of bytes below which blocks will be encoded
+// directly into CID instead of being stored and addressed by it's hash.
+// Specifying this option won't enable block inlining. For that use `Inline`
+// option. Default: 32 bytes
+//
+// Note that while there is no hard limit on the number of bytes, it should be
+// kept at a reasonably low value, such as 64; implementations may choose to
+// reject anything larger.
+func (unixfsOpts) InlineLimit(limit int) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.InlineLimit = limit
+		return nil
+	}
+}
+
+// Chunker specifies settings for the chunking algorithm to use.
+//
+// Default: size-262144, formats:
+// size-[bytes] - Simple chunker splitting data into blocks of n bytes
+// rabin-[min]-[avg]-[max] - Rabin chunker
+func (unixfsOpts) Chunker(chunker string) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Chunker = chunker
+		return nil
+	}
+}
+
+// Layout tells the adder how to balance data between leaves.
+// options.BalancedLayout is the default, it's optimized for static seekable
+// files.
+// options.TrickleLayout is optimized for streaming data,
+func (unixfsOpts) Layout(layout Layout) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Layout = layout
+		return nil
+	}
+}
+
+// Pin tells the adder to pin the file root recursively after adding
+func (unixfsOpts) Pin(pin bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Pin = pin
+		return nil
+	}
+}
+
+// HashOnly will make the adder calculate data hash without storing it in the
+// blockstore or announcing it to the network
+func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.OnlyHash = hashOnly
+		return nil
+	}
+}
+
+// Local will add the data to blockstore without announcing it to the network
+//
+// Note that this doesn't prevent other nodes from getting this data
+func (unixfsOpts) Local(local bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Local = local
+		return nil
+	}
+}
+
+// Wrap tells the adder to wrap the added file structure with an additional
+// directory.
+func (unixfsOpts) Wrap(wrap bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Wrap = wrap
+		return nil
+	}
+}
+
+// Hidden enables adding of hidden files (files prefixed with '.')
+func (unixfsOpts) Hidden(hidden bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Hidden = hidden
+		return nil
+	}
+}
+
+// StdinName is the name set for files which don specify FilePath as
+// os.Stdin.Name()
+func (unixfsOpts) StdinName(name string) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.StdinName = name
+		return nil
+	}
+}
+
+// Events specifies channel which will be used to report events about ongoing
+// Add operation.
+//
+// Note that if this channel blocks it may slowdown the adder
+func (unixfsOpts) Events(sink chan<- interface{}) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Events = sink
+		return nil
+	}
+}
+
+// Silent reduces event output
+func (unixfsOpts) Silent(silent bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Silent = silent
+		return nil
+	}
+}
+
+// Progress tells the adder whether to enable progress events
+func (unixfsOpts) Progress(enable bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.Progress = enable
+		return nil
+	}
+}
+
+// FsCache tells the adder to check the filestore for pre-existing blocks
+//
+// Experimental
+func (unixfsOpts) FsCache(enable bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.FsCache = enable
+		return nil
+	}
+}
+
+// NoCopy tells the adder to add the files using filestore. Implies RawLeaves.
+//
+// Experimental
+func (unixfsOpts) Nocopy(enable bool) UnixfsAddOption {
+	return func(settings *UnixfsAddSettings) error {
+		settings.NoCopy = enable
+		return nil
+	}
+}

path.go

 package iface
 
 import (
-	ipfspath "gx/ipfs/QmcjwUb36Z16NJkvDX6ccXPqsFswo6AsRXynyXcLLCphV2/go-path"
+	ipfspath "gx/ipfs/QmQmMu1vsgsjxyB8tzrA6ZTCTCLDLVaXMb4Q57r2v886Sx/go-path"
 
 	cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
 )

pubsub.go

+package iface
+
+import (
+	"context"
+	"io"
+
+	options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer"
+)
+
+// PubSubSubscription is an active PubSub subscription
+type PubSubSubscription interface {
+	io.Closer
+
+	// Next return the next incoming message
+	Next(context.Context) (PubSubMessage, error)
+}
+
+// PubSubMessage is a single PubSub message
+type PubSubMessage interface {
+	// From returns id of a peer from which the message has arrived
+	From() peer.ID
+
+	// Data returns the message body
+	Data() []byte
+
+	// Seq returns message identifier
+	Seq() []byte
+
+	// Topics returns list of topics this message was set to
+	Topics() []string
+}
+
+// PubSubAPI specifies the interface to PubSub
+type PubSubAPI interface {
+	// Ls lists subscribed topics by name
+	Ls(context.Context) ([]string, error)
+
+	// Peers list peers we are currently pubsubbing with
+	Peers(context.Context, ...options.PubSubPeersOption) ([]peer.ID, error)
+
+	// Publish a message to a given pubsub topic
+	Publish(context.Context, string, []byte) error
+
+	// Subscribe to messages on a given topic
+	Subscribe(context.Context, string, ...options.PubSubSubscribeOption) (PubSubSubscription, error)
+}

swarm.go

+package iface
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	net "gx/ipfs/QmWUPYHpNv4YahaBYXovuEJttgfqcNcN9Gg4arhQYcRoqa/go-libp2p-net"
+	pstore "gx/ipfs/QmXEyLwySuDMXejWBu8XwdkX2WuGKk8x9jFwz8js7j72UX/go-libp2p-peerstore"
+	ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr"
+	"gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
+	"gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer"
+)
+
+var (
+	ErrNotConnected = errors.New("not connected")
+	ErrConnNotFound = errors.New("conn not found")
+)
+
+// ConnectionInfo contains information about a peer
+type ConnectionInfo interface {
+	// ID returns PeerID
+	ID() peer.ID
+
+	// Address returns the multiaddress via which we are connected with the peer
+	Address() ma.Multiaddr
+
+	// Direction returns which way the connection was established
+	Direction() net.Direction
+
+	// Latency returns last known round trip time to the peer
+	Latency() (time.Duration, error)
+
+	// Streams returns list of streams established with the peer
+	Streams() ([]protocol.ID, error)
+}
+
+// SwarmAPI specifies the interface to libp2p swarm
+type SwarmAPI interface {
+	// Connect to a given peer
+	Connect(context.Context, pstore.PeerInfo) error
+
+	// Disconnect from a given address
+	Disconnect(context.Context, ma.Multiaddr) error
+
+	// Peers returns the list of peers we are connected to
+	Peers(context.Context) ([]ConnectionInfo, error)
+
+	// KnownAddrs returns the list of all addresses this node is aware of
+	KnownAddrs(context.Context) (map[peer.ID][]ma.Multiaddr, error)
+
+	// LocalAddrs returns the list of announced listening addresses
+	LocalAddrs(context.Context) ([]ma.Multiaddr, error)
+
+	// ListenAddrs returns the list of all listening addresses
+	ListenAddrs(context.Context) ([]ma.Multiaddr, error)
+}

unixfs.go

@@ -2,17 +2,37 @@ package iface
 
 import (
 	"context"
-	"io"
 
+	options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
+
+	files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files"
 	ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format"
 )
 
+// TODO: ideas on making this more coreapi-ish without breaking the http API?
+type AddEvent struct {
+	Name  string
+	Hash  string `json:",omitempty"`
+	Bytes int64  `json:",omitempty"`
+	Size  string `json:",omitempty"`
+}
+
 // UnixfsAPI is the basic interface to immutable files in IPFS
+// NOTE: This API is heavily WIP, things are guaranteed to break frequently
 type UnixfsAPI interface {
 	// Add imports the data from the reader into merkledag file
-	Add(context.Context, io.Reader) (ResolvedPath, error)
+	//
+	// TODO: a long useful comment on how to use this for many different scenarios
+	Add(context.Context, files.File, ...options.UnixfsAddOption) (ResolvedPath, error)
+
+	// Get returns a read-only handle to a file tree referenced by a path
+	//
+	// Note that some implementations of this API may apply the specified context
+	// to operations performed on the returned file
+	Get(context.Context, Path) (files.File, error)
 
 	// Cat returns a reader for the file
+	// TODO: Remove in favour of Get (if we use Get on a file we still have reader directly, so..)
 	Cat(context.Context, Path) (Reader, error)
 
 	// Ls returns the list of links in a directory

util.go

 package iface
 
 import (
+	"context"
 	"io"
 )
 
 type Reader interface {
-	io.ReadSeeker
+	ReadSeekCloser
+	Size() uint64
+	CtxReadFull(context.Context, []byte) (int, error)
+}
+
+// A ReadSeekCloser implements interfaces to read, copy, seek and close.
+type ReadSeekCloser interface {
+	io.Reader
+	io.Seeker
 	io.Closer
+	io.WriterTo
 }