Merge branch 'port-2021-05-11'

.github/workflows/automerge.yml

+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+# Automatically merge pull requests opened by web3-bot, as soon as (and only if) all tests pass.
+# This reduces the friction associated with updating with our workflows.
+
+on: [ pull_request ]
+
+jobs:
+  automerge:
+    if: github.event.pull_request.user.login == 'web3-bot'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Wait on tests
+      uses: lewagon/wait-on-check-action@bafe56a6863672c681c3cf671f5e10b20abf2eaa # v0.2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        wait-interval: 10
+        running-workflow-name: 'automerge' # the name of this job
+    - name: Merge PR
+      uses: pascalgn/automerge-action@741c311a47881be9625932b0a0de1b0937aab1ae # v0.13.1
+      env:
+        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        MERGE_LABELS: ""
+        MERGE_METHOD: "squash"
+        MERGE_DELETE_BRANCH: true

.github/workflows/go-check.yml

+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+on: [push, pull_request]
+
+jobs:
+  unit:
+    runs-on: ubuntu-latest
+    name: Go checks
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: "1.16.x"
+      - name: Install staticcheck
+        run: go install honnef.co/go/tools/cmd/staticcheck@be534f007836a777104a15f2456cd1fffd3ddee8 # v2020.2.2
+      - name: Check that go.mod is tidy
+        run: |
+          go mod tidy
+          if [[ -n $(git ls-files --other --exclude-standard --directory -- go.sum) ]]; then
+            echo "go.sum was added by go mod tidy"
+            exit 1
+          fi
+          git diff --exit-code -- go.sum go.mod
+      - name: gofmt
+        if: ${{ success() || failure() }} # run this step even if the previous one failed
+        run: |
+          out=$(gofmt -s -l .)
+          if [[ -n "$out" ]]; then
+            echo $out | awk '{print "::error file=" $0 ",line=0,col=0::File is not gofmt-ed."}'
+            exit 1
+          fi
+      - name: go vet
+        if: ${{ success() || failure() }} # run this step even if the previous one failed
+        run: go vet ./...
+      - name: staticcheck
+        if: ${{ success() || failure() }} # run this step even if the previous one failed
+        run: |
+          set -o pipefail
+          staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g'
+

.github/workflows/go-test.yml

+# File managed by web3-bot. DO NOT EDIT.
+# See https://github.com/protocol/.github/ for details.
+
+on: [push, pull_request]
+
+jobs:
+  unit:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ "ubuntu", "windows", "macos" ]
+        go: [ "1.15.x", "1.16.x" ]
+    runs-on: ${{ matrix.os }}-latest
+    name: Unit tests (${{ matrix.os}}, Go ${{ matrix.go }})
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Go information
+        run: |
+          go version
+          go env
+      - name: Run tests
+        run: go test -v -coverprofile coverage.txt ./...
+      - name: Run tests (32 bit)
+        if: ${{ matrix.os != 'macos' }} # can't run 32 bit tests on OSX.
+        env:
+          GOARCH: 386
+        run: go test -v ./...
+      - name: Run tests with race detector
+        if: ${{ matrix.os == 'ubuntu' }} # speed things up. Windows and OSX VMs are slow
+        run: go test -v -race ./...
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@967e2b38a85a62bd61be5529ada27ebc109948c2 # v1.4.1
+        with:
+          file: coverage.txt
+          env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }}

LICENSE

+This project is transitioning from an MIT-only license to a dual MIT/Apache-2.0 license. 
+Unless otherwise noted, all code contributed prior to 2019-05-06 and not contributed by 
+a user listed in [this signoff issue](https://github.com/ipfs/go-ipfs/issues/6302) is 
+licensed under MIT-only. All new contributions (and past contributions since 2019-05-06) 
+are licensed under a dual MIT/Apache-2.0 license.
+
+MIT: https://www.opensource.org/licenses/mit
+Apache-2.0: https://www.apache.org/licenses/license-2.0

LICENSE-APACHE

+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

LICENSE-MIT

+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

 # go-namesys
 
+[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://protocol.ai)
+[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
+[![Go Reference](https://pkg.go.dev/badge/github.com/ipfs/go-namesys.svg)](https://pkg.go.dev/github.com/ipfs/go-namesys)
+[![Travis CI](https://travis-ci.com/ipfs/go-namesys.svg?branch=master)](https://travis-ci.com/ipfs/go-namesys)
+
+
+> go-namesys provides publish and resolution support for the /ipns/ namespace
+
+Package namesys defines `Resolver` and `Publisher` interfaces for IPNS paths, that is, paths in the form of `/ipns/<name_to_be_resolved>`. A "resolved" IPNS path becomes an `/ipfs/<cid>` path.
+
+Traditionally, these paths would be in the form of `/ipns/{libp2p-key}`, which references an IPNS record in a distributed `ValueStore` (usually the IPFS DHT).
+
+Additionally, the `/ipns/` namespace can also be used with domain names that use DNSLink (`/ipns/en.wikipedia-on-ipfs.org`, see https://docs.ipfs.io/concepts/dnslink/).
+
+The package provides implementations for all three resolvers.
+
+## Table of Contents
+
+- [Install](#install)
+- [Usage](#usage)
+- [Contribute](#contribute)
+- [License](#license)
+
+## Install
+
+`go-namesys` works like a regular Go module:
+```
+> go get github.com/ipfs/go-namesys
+```
+
+## Usage
+```
+import "github.com/ipfs/go-namesys"
+```
+
+See the [Pkg.go.dev documentation](https://pkg.go.dev/github.com/ipfs/go-namesys)
+
+## Contribute
+
+PRs accepted.
+
+Small note: If editing the README, please conform to the [standard-readme](https://github.com/RichardLitt/standard-readme) specification.
+
+## License
+
+This project is dual-licensed under Apache 2.0 and MIT terms:
+
+- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0)
+- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT)

base.go

+package namesys
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	path "gitlab.dms3.io/dms3/go-path"
+	opts "gitlab.dms3.io/dms3/interface-go-dms3-core/options/namesys"
+)
+
+type onceResult struct {
+	value path.Path
+	ttl   time.Duration
+	err   error
+}
+
+type resolver interface {
+	resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult
+}
+
+// resolve is a helper for implementing Resolver.ResolveN using resolveOnce.
+func resolve(ctx context.Context, r resolver, name string, options opts.ResolveOpts) (path.Path, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	err := ErrResolveFailed
+	var p path.Path
+
+	resCh := resolveAsync(ctx, r, name, options)
+
+	for res := range resCh {
+		p, err = res.Path, res.Err
+		if err != nil {
+			break
+		}
+	}
+
+	return p, err
+}
+
+func resolveAsync(ctx context.Context, r resolver, name string, options opts.ResolveOpts) <-chan Result {
+	resCh := r.resolveOnceAsync(ctx, name, options)
+	depth := options.Depth
+	outCh := make(chan Result, 1)
+
+	go func() {
+		defer close(outCh)
+		var subCh <-chan Result
+		var cancelSub context.CancelFunc
+		defer func() {
+			if cancelSub != nil {
+				cancelSub()
+			}
+		}()
+
+		for {
+			select {
+			case res, ok := <-resCh:
+				if !ok {
+					resCh = nil
+					break
+				}
+
+				if res.err != nil {
+					emitResult(ctx, outCh, Result{Err: res.err})
+					return
+				}
+				log.Debugf("resolved %s to %s", name, res.value.String())
+				if !strings.HasPrefix(res.value.String(), dms3nsPrefix) {
+					emitResult(ctx, outCh, Result{Path: res.value})
+					break
+				}
+
+				if depth == 1 {
+					emitResult(ctx, outCh, Result{Path: res.value, Err: ErrResolveRecursion})
+					break
+				}
+
+				subopts := options
+				if subopts.Depth > 1 {
+					subopts.Depth--
+				}
+
+				var subCtx context.Context
+				if cancelSub != nil {
+					// Cancel previous recursive resolve since it won't be used anyways
+					cancelSub()
+				}
+				subCtx, cancelSub = context.WithCancel(ctx)
+				_ = cancelSub
+
+				p := strings.TrimPrefix(res.value.String(), dms3nsPrefix)
+				subCh = resolveAsync(subCtx, r, p, subopts)
+			case res, ok := <-subCh:
+				if !ok {
+					subCh = nil
+					break
+				}
+
+				// We don't bother returning here in case of context timeout as there is
+				// no good reason to do that, and we may still be able to emit a result
+				emitResult(ctx, outCh, res)
+			case <-ctx.Done():
+				return
+			}
+			if resCh == nil && subCh == nil {
+				return
+			}
+		}
+	}()
+	return outCh
+}
+
+func emitResult(ctx context.Context, outCh chan<- Result, r Result) {
+	select {
+	case outCh <- r:
+	case <-ctx.Done():
+	}
+}

cache.go

+package namesys
+
+import (
+	"time"
+
+	path "gitlab.dms3.io/dms3/go-path"
+)
+
+func (ns *mpns) cacheGet(name string) (path.Path, bool) {
+	// existence of optional mapping defined via DMS3_NS_MAP is checked first
+	if ns.staticMap != nil {
+		val, ok := ns.staticMap[name]
+		if ok {
+			return val, true
+		}
+	}
+
+	if ns.cache == nil {
+		return "", false
+	}
+
+	ientry, ok := ns.cache.Get(name)
+	if !ok {
+		return "", false
+	}
+
+	entry, ok := ientry.(cacheEntry)
+	if !ok {
+		// should never happen, purely for sanity
+		log.Panicf("unexpected type %T in cache for %q.", ientry, name)
+	}
+
+	if time.Now().Before(entry.eol) {
+		return entry.val, true
+	}
+
+	ns.cache.Remove(name)
+
+	return "", false
+}
+
+func (ns *mpns) cacheSet(name string, val path.Path, ttl time.Duration) {
+	if ns.cache == nil || ttl <= 0 {
+		return
+	}
+	ns.cache.Add(name, cacheEntry{
+		val: val,
+		eol: time.Now().Add(ttl),
+	})
+}
+
+func (ns *mpns) cacheInvalidate(name string) {
+	if ns.cache == nil {
+		return
+	}
+	ns.cache.Remove(name)
+}
+
+type cacheEntry struct {
+	val path.Path
+	eol time.Time
+}

dms3ns_resolver_validation_test.go

+package namesys
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	ds "gitlab.dms3.io/dms3/go-datastore"
+	dssync "gitlab.dms3.io/dms3/go-datastore/sync"
+	mockrouting "gitlab.dms3.io/dms3/go-dms3-routing/mock"
+	offline "gitlab.dms3.io/dms3/go-dms3-routing/offline"
+	dms3ns "gitlab.dms3.io/dms3/go-dms3ns"
+	dms3ns_pb "gitlab.dms3.io/dms3/go-dms3ns/pb"
+	path "gitlab.dms3.io/dms3/go-path"
+	opts "gitlab.dms3.io/dms3/interface-go-dms3-core/options/namesys"
+	ci "gitlab.dms3.io/p2p/go-p2p-core/crypto"
+	peer "gitlab.dms3.io/p2p/go-p2p-core/peer"
+	pstore "gitlab.dms3.io/p2p/go-p2p-core/peerstore"
+	routing "gitlab.dms3.io/p2p/go-p2p-core/routing"
+	"gitlab.dms3.io/p2p/go-p2p-core/test"
+	pstoremem "gitlab.dms3.io/p2p/go-p2p-peerstore/pstoremem"
+	record "gitlab.dms3.io/p2p/go-p2p-record"
+	testutil "gitlab.dms3.io/p2p/go-p2p-testing/net"
+)
+
+func TestResolverValidation(t *testing.T) {
+	t.Run("RSA",
+		func(t *testing.T) {
+			testResolverValidation(t, ci.RSA)
+		})
+	t.Run("Ed25519",
+		func(t *testing.T) {
+			testResolverValidation(t, ci.Ed25519)
+		})
+	t.Run("ECDSA",
+		func(t *testing.T) {
+			testResolverValidation(t, ci.ECDSA)
+		})
+	t.Run("Secp256k1",
+		func(t *testing.T) {
+			testResolverValidation(t, ci.Secp256k1)
+		})
+}
+
+func testResolverValidation(t *testing.T, keyType int) {
+	ctx := context.Background()
+	rid := testutil.RandIdentityOrFatal(t)
+	dstore := dssync.MutexWrap(ds.NewMapDatastore())
+	peerstore := pstoremem.NewPeerstore()
+
+	vstore := newMockValueStore(rid, dstore, peerstore)
+	resolver := NewDms3NsResolver(vstore)
+
+	nvVstore := offline.NewOfflineRouter(dstore, mockrouting.MockValidator{})
+
+	// Create entry with expiry in one hour
+	priv, id, _, dms3nsDHTPath := genKeys(t, keyType)
+	ts := time.Now()
+	p := []byte("/dms3/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG")
+	entry, err := createDMS3NSRecordWithEmbeddedPublicKey(priv, p, 1, ts.Add(time.Hour))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Publish entry
+	err = PublishEntry(ctx, vstore, dms3nsDHTPath, entry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Resolve entry
+	resp, err := resolve(ctx, resolver, id.Pretty(), opts.DefaultResolveOpts())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp != path.Path(p) {
+		t.Fatalf("Mismatch between published path %s and resolved path %s", p, resp)
+	}
+	// Create expired entry
+	expiredEntry, err := createDMS3NSRecordWithEmbeddedPublicKey(priv, p, 1, ts.Add(-1*time.Hour))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Publish entry
+	err = PublishEntry(ctx, nvVstore, dms3nsDHTPath, expiredEntry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Record should fail validation because entry is expired
+	_, err = resolve(ctx, resolver, id.Pretty(), opts.DefaultResolveOpts())
+	if err == nil {
+		t.Fatal("ValidateDms3NsRecord should have returned error")
+	}
+
+	// Create DMS3NS record path with a different private key
+	priv2, id2, _, dms3nsDHTPath2 := genKeys(t, keyType)
+
+	// Publish entry
+	err = PublishEntry(ctx, nvVstore, dms3nsDHTPath2, entry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Record should fail validation because public key defined by
+	// dms3ns path doesn't match record signature
+	_, err = resolve(ctx, resolver, id2.Pretty(), opts.DefaultResolveOpts())
+	if err == nil {
+		t.Fatal("ValidateDms3NsRecord should have failed signature verification")
+	}
+
+	// Try embedding the incorrect private key inside the entry
+	if err := dms3ns.EmbedPublicKey(priv2.GetPublic(), entry); err != nil {
+		t.Fatal(err)
+	}
+
+	// Publish entry
+	err = PublishEntry(ctx, nvVstore, dms3nsDHTPath2, entry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Record should fail validation because public key defined by
+	// dms3ns path doesn't match record signature
+	_, err = resolve(ctx, resolver, id2.Pretty(), opts.DefaultResolveOpts())
+	if err == nil {
+		t.Fatal("ValidateDms3NsRecord should have failed signature verification")
+	}
+}
+
+func genKeys(t *testing.T, keyType int) (ci.PrivKey, peer.ID, string, string) {
+	bits := 0
+	if keyType == ci.RSA {
+		bits = 2048
+	}
+
+	sk, pk, err := test.RandTestKeyPair(keyType, bits)
+	if err != nil {
+		t.Fatal(err)
+	}
+	id, err := peer.IDFromPublicKey(pk)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return sk, id, PkKeyForID(id), dms3ns.RecordKey(id)
+}
+
+func createDMS3NSRecordWithEmbeddedPublicKey(sk ci.PrivKey, val []byte, seq uint64, eol time.Time) (*dms3ns_pb.Dms3NsEntry, error) {
+	entry, err := dms3ns.Create(sk, val, seq, eol)
+	if err != nil {
+		return nil, err
+	}
+	if err := dms3ns.EmbedPublicKey(sk.GetPublic(), entry); err != nil {
+		return nil, err
+	}
+
+	return entry, nil
+}
+
+type mockValueStore struct {
+	r     routing.ValueStore
+	kbook pstore.KeyBook
+}
+
+func newMockValueStore(id testutil.Identity, dstore ds.Datastore, kbook pstore.KeyBook) *mockValueStore {
+	return &mockValueStore{
+		r: offline.NewOfflineRouter(dstore, record.NamespacedValidator{
+			"dms3ns": dms3ns.Validator{KeyBook: kbook},
+			"pk":     record.PublicKeyValidator{},
+		}),
+		kbook: kbook,
+	}
+}
+
+func (m *mockValueStore) GetValue(ctx context.Context, k string, opts ...routing.Option) ([]byte, error) {
+	return m.r.GetValue(ctx, k, opts...)
+}
+
+func (m *mockValueStore) SearchValue(ctx context.Context, k string, opts ...routing.Option) (<-chan []byte, error) {
+	return m.r.SearchValue(ctx, k, opts...)
+}
+
+func (m *mockValueStore) GetPublicKey(ctx context.Context, p peer.ID) (ci.PubKey, error) {
+	pk := m.kbook.PubKey(p)
+	if pk != nil {
+		return pk, nil
+	}
+
+	pkkey := routing.KeyForPublicKey(p)
+	val, err := m.GetValue(ctx, pkkey)
+	if err != nil {
+		return nil, err
+	}
+
+	pk, err = ci.UnmarshalPublicKey(val)
+	if err != nil {
+		return nil, err
+	}
+
+	return pk, m.kbook.AddPubKey(p, pk)
+}
+
+func (m *mockValueStore) PutValue(ctx context.Context, k string, d []byte, opts ...routing.Option) error {
+	return m.r.PutValue(ctx, k, d, opts...)
+}

dns.go

+package namesys
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	gpath "path"
+	"strings"
+
+	dns "github.com/miekg/dns"
+	path "gitlab.dms3.io/dms3/go-path"
+	opts "gitlab.dms3.io/dms3/interface-go-dms3-core/options/namesys"
+)
+
+// LookupTXTFunc is a function that lookups TXT record values.
+type LookupTXTFunc func(ctx context.Context, name string) (txt []string, err error)
+
+// DNSResolver implements a Resolver on DNS domains
+type DNSResolver struct {
+	lookupTXT LookupTXTFunc
+	// TODO: maybe some sort of caching?
+	// cache would need a timeout
+}
+
+// NewDNSResolver constructs a name resolver using DNS TXT records.
+func NewDNSResolver(lookup LookupTXTFunc) *DNSResolver {
+	return &DNSResolver{lookupTXT: lookup}
+}
+
+// Resolve implements Resolver.
+func (r *DNSResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) {
+	return resolve(ctx, r, name, opts.ProcessOpts(options))
+}
+
+// ResolveAsync implements Resolver.
+func (r *DNSResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result {
+	return resolveAsync(ctx, r, name, opts.ProcessOpts(options))
+}
+
+type lookupRes struct {
+	path  path.Path
+	error error
+}
+
+// resolveOnce implements resolver.
+// TXT records for a given domain name should contain a b58
+// encoded multihash.
+func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
+	var fqdn string
+	out := make(chan onceResult, 1)
+	segments := strings.SplitN(name, "/", 2)
+	domain := segments[0]
+
+	if _, ok := dns.IsDomainName(domain); !ok {
+		out <- onceResult{err: fmt.Errorf("not a valid domain name: %s", domain)}
+		close(out)
+		return out
+	}
+	log.Debugf("DNSResolver resolving %s", domain)
+
+	if strings.HasSuffix(domain, ".") {
+		fqdn = domain
+	} else {
+		fqdn = domain + "."
+	}
+
+	rootChan := make(chan lookupRes, 1)
+	go workDomain(ctx, r, fqdn, rootChan)
+
+	subChan := make(chan lookupRes, 1)
+	go workDomain(ctx, r, "_dnslink."+fqdn, subChan)
+
+	appendPath := func(p path.Path) (path.Path, error) {
+		if len(segments) > 1 {
+			return path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[1])
+		}
+		return p, nil
+	}
+
+	go func() {
+		defer close(out)
+		var rootResErr, subResErr error
+		for {
+			select {
+			case subRes, ok := <-subChan:
+				if !ok {
+					subChan = nil
+					break
+				}
+				if subRes.error == nil {
+					p, err := appendPath(subRes.path)
+					emitOnceResult(ctx, out, onceResult{value: p, err: err})
+					// Return without waiting for rootRes, since this result
+					// (for "_dnslink."+fqdn) takes precedence
+					return
+				}
+				subResErr = subRes.error
+			case rootRes, ok := <-rootChan:
+				if !ok {
+					rootChan = nil
+					break
+				}
+				if rootRes.error == nil {
+					p, err := appendPath(rootRes.path)
+					emitOnceResult(ctx, out, onceResult{value: p, err: err})
+					// Do not return here.  Wait for subRes so that it is
+					// output last if good, thereby giving subRes precedence.
+				} else {
+					rootResErr = rootRes.error
+				}
+			case <-ctx.Done():
+				return
+			}
+			if subChan == nil && rootChan == nil {
+				// If here, then both lookups are done
+				//
+				// If both lookups failed due to no TXT records with a
+				// dnslink, then output a more specific error message
+				if rootResErr == ErrResolveFailed && subResErr == ErrResolveFailed {
+					// Wrap error so that it can be tested if it is a ErrResolveFailed
+					err := fmt.Errorf("%w: %q is missing a DNSLink record (https://docs.dms3.io/concepts/dnslink/)", ErrResolveFailed, gpath.Base(name))
+					emitOnceResult(ctx, out, onceResult{err: err})
+				}
+				return
+			}
+		}
+	}()
+
+	return out
+}
+
+func workDomain(ctx context.Context, r *DNSResolver, name string, res chan lookupRes) {
+	defer close(res)
+
+	txt, err := r.lookupTXT(ctx, name)
+	if err != nil {
+		if dnsErr, ok := err.(*net.DNSError); ok {
+			// If no TXT records found, return same error as when no text
+			// records contain dnslink. Otherwise, return the actual error.
+			if dnsErr.IsNotFound {
+				err = ErrResolveFailed
+			}
+		}
+		// Could not look up any text records for name
+		res <- lookupRes{"", err}
+		return
+	}
+
+	for _, t := range txt {
+		p, err := parseEntry(t)
+		if err == nil {
+			res <- lookupRes{p, nil}
+			return
+		}
+	}
+
+	// There were no TXT records with a dnslink
+	res <- lookupRes{"", ErrResolveFailed}
+}
+
+func parseEntry(txt string) (path.Path, error) {
+	p, err := path.ParseCidToPath(txt) // bare DMS3 multihashes
+	if err == nil {
+		return p, nil
+	}
+
+	return tryParseDNSLink(txt)
+}
+
+func tryParseDNSLink(txt string) (path.Path, error) {
+	parts := strings.SplitN(txt, "=", 2)
+	if len(parts) == 2 && parts[0] == "dnslink" {
+		return path.ParsePath(parts[1])
+	}
+
+	return "", errors.New("not a valid dnslink entry")
+}

dns_test.go

+package namesys
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	opts "gitlab.dms3.io/dms3/interface-go-dms3-core/options/namesys"
+)
+
+type mockDNS struct {
+	entries map[string][]string
+}
+
+func (m *mockDNS) lookupTXT(ctx context.Context, name string) (txt []string, err error) {
+	txt, ok := m.entries[name]
+	if !ok {
+		return nil, fmt.Errorf("no TXT entry for %s", name)
+	}
+	return txt, nil
+}
+
+func TestDnsEntryParsing(t *testing.T) {
+	goodEntries := []string{
+		"QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+		"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+		"dnslink=/dms3ns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+		"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo",
+		"dnslink=/dms3ns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar",
+		"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz",
+		"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz/",
+		"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+	}
+
+	badEntries := []string{
+		"QmYhE8xgFCjGcz6PHgnvJz5NOTCORRECT",
+		"quux=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+		"dnslink=",
+		"dnslink=/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo",
+		"dnslink=dms3ns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar",
+	}
+
+	for _, e := range goodEntries {
+		_, err := parseEntry(e)
+		if err != nil {
+			t.Log("expected entry to parse correctly!")
+			t.Log(e)
+			t.Fatal(err)
+		}
+	}
+
+	for _, e := range badEntries {
+		_, err := parseEntry(e)
+		if err == nil {
+			t.Log("expected entry parse to fail!")
+			t.Fatal(err)
+		}
+	}
+}
+
+func newMockDNS() *mockDNS {
+	return &mockDNS{
+		entries: map[string][]string{
+			"multihash.example.com.": {
+				"dnslink=QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+			},
+			"dms3.example.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+			},
+			"_dnslink.ddms3.example.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+			},
+			"dns1.example.com.": {
+				"dnslink=/dms3ns/dms3.example.com",
+			},
+			"dns2.example.com.": {
+				"dnslink=/dms3ns/dns1.example.com",
+			},
+			"multi.example.com.": {
+				"some stuff",
+				"dnslink=/dms3ns/dns1.example.com",
+				"masked dnslink=/dms3ns/example.invalid",
+			},
+			"equals.example.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals",
+			},
+			"loop1.example.com.": {
+				"dnslink=/dms3ns/loop2.example.com",
+			},
+			"loop2.example.com.": {
+				"dnslink=/dms3ns/loop1.example.com",
+			},
+			"_dnslink.dloop1.example.com.": {
+				"dnslink=/dms3ns/loop2.example.com",
+			},
+			"_dnslink.dloop2.example.com.": {
+				"dnslink=/dms3ns/loop1.example.com",
+			},
+			"bad.example.com.": {
+				"dnslink=",
+			},
+			"withsegment.example.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment",
+			},
+			"withrecsegment.example.com.": {
+				"dnslink=/dms3ns/withsegment.example.com/subsub",
+			},
+			"withtrailing.example.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/",
+			},
+			"withtrailingrec.example.com.": {
+				"dnslink=/dms3ns/withtrailing.example.com/segment/",
+			},
+			"double.example.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+			},
+			"_dnslink.double.example.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+			},
+			"double.conflict.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD",
+			},
+			"_dnslink.conflict.example.com.": {
+				"dnslink=/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE",
+			},
+			"fqdn.example.com.": {
+				"dnslink=/dms3/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr",
+			},
+			"en.wikipedia-on-dms3.org.": {
+				"dnslink=/dms3/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze",
+			},
+			"custom.non-icann.tldextravaganza.": {
+				"dnslink=/dms3/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm",
+			},
+			"singlednslabelshouldbeok.": {
+				"dnslink=/dms3/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi",
+			},
+			"www.wealdtech.eth.": {
+				"dnslink=/dms3ns/dms3.example.com",
+			},
+		},
+	}
+}
+
+func TestDNSResolution(t *testing.T) {
+	mock := newMockDNS()
+	r := &DNSResolver{lookupTXT: mock.lookupTXT}
+	testResolution(t, r, "multihash.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+	testResolution(t, r, "dms3.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+	testResolution(t, r, "ddms3.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+	testResolution(t, r, "dns1.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+	testResolution(t, r, "dns1.example.com", 1, "/dms3ns/dms3.example.com", ErrResolveRecursion)
+	testResolution(t, r, "dns2.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+	testResolution(t, r, "dns2.example.com", 1, "/dms3ns/dns1.example.com", ErrResolveRecursion)
+	testResolution(t, r, "dns2.example.com", 2, "/dms3ns/dms3.example.com", ErrResolveRecursion)
+	testResolution(t, r, "multi.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+	testResolution(t, r, "multi.example.com", 1, "/dms3ns/dns1.example.com", ErrResolveRecursion)
+	testResolution(t, r, "multi.example.com", 2, "/dms3ns/dms3.example.com", ErrResolveRecursion)
+	testResolution(t, r, "equals.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", nil)
+	testResolution(t, r, "loop1.example.com", 1, "/dms3ns/loop2.example.com", ErrResolveRecursion)
+	testResolution(t, r, "loop1.example.com", 2, "/dms3ns/loop1.example.com", ErrResolveRecursion)
+	testResolution(t, r, "loop1.example.com", 3, "/dms3ns/loop2.example.com", ErrResolveRecursion)
+	testResolution(t, r, "loop1.example.com", opts.DefaultDepthLimit, "/dms3ns/loop1.example.com", ErrResolveRecursion)
+	testResolution(t, r, "dloop1.example.com", 1, "/dms3ns/loop2.example.com", ErrResolveRecursion)
+	testResolution(t, r, "dloop1.example.com", 2, "/dms3ns/loop1.example.com", ErrResolveRecursion)
+	testResolution(t, r, "dloop1.example.com", 3, "/dms3ns/loop2.example.com", ErrResolveRecursion)
+	testResolution(t, r, "dloop1.example.com", opts.DefaultDepthLimit, "/dms3ns/loop1.example.com", ErrResolveRecursion)
+	testResolution(t, r, "bad.example.com", opts.DefaultDepthLimit, "", ErrResolveFailed)
+	testResolution(t, r, "withsegment.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", nil)
+	testResolution(t, r, "withrecsegment.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub", nil)
+	testResolution(t, r, "withsegment.example.com/test1", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/test1", nil)
+	testResolution(t, r, "withrecsegment.example.com/test2", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test2", nil)
+	testResolution(t, r, "withrecsegment.example.com/test3/", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test3/", nil)
+	testResolution(t, r, "withtrailingrec.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/", nil)
+	testResolution(t, r, "double.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+	testResolution(t, r, "conflict.example.com", opts.DefaultDepthLimit, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", nil)
+	testResolution(t, r, "fqdn.example.com.", opts.DefaultDepthLimit, "/dms3/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", nil)
+	testResolution(t, r, "en.wikipedia-on-dms3.org", 2, "/dms3/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", nil)
+	testResolution(t, r, "custom.non-icann.tldextravaganza.", 2, "/dms3/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", nil)
+	testResolution(t, r, "singlednslabelshouldbeok", 2, "/dms3/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", nil)
+	testResolution(t, r, "www.wealdtech.eth", 1, "/dms3ns/dms3.example.com", ErrResolveRecursion)
+	testResolution(t, r, "www.wealdtech.eth", 2, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+	testResolution(t, r, "www.wealdtech.eth", 2, "/dms3/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil)
+}

go.mod

+module gitlab.dms3.io/dms3/go-namesys
+
+require (
+	github.com/gogo/protobuf v1.3.2
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/jbenet/goprocess v0.1.4
+	github.com/miekg/dns v1.1.41
+	github.com/multiformats/go-multihash v0.0.15
+	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc
+	gitlab.dms3.io/dms3/go-cid v0.0.4
+	gitlab.dms3.io/dms3/go-datastore v0.0.2
+	gitlab.dms3.io/dms3/go-dms3-ds-help v0.0.3
+	gitlab.dms3.io/dms3/go-dms3-keystore v0.0.2
+	gitlab.dms3.io/dms3/go-dms3-routing v0.0.2
+	gitlab.dms3.io/dms3/go-dms3ns v0.0.2
+	gitlab.dms3.io/dms3/go-log v0.0.2
+	gitlab.dms3.io/dms3/go-path v0.0.1
+	gitlab.dms3.io/dms3/interface-go-dms3-core v0.0.1
+	gitlab.dms3.io/mf/go-multiaddr v0.0.2
+	gitlab.dms3.io/mf/go-multiaddr-dns v0.0.1
+	gitlab.dms3.io/p2p/go-p2p v0.0.4
+	gitlab.dms3.io/p2p/go-p2p-core v0.0.2
+	gitlab.dms3.io/p2p/go-p2p-kad-dht v0.0.1
+	gitlab.dms3.io/p2p/go-p2p-peerstore v0.0.2
+	gitlab.dms3.io/p2p/go-p2p-record v0.0.2
+	gitlab.dms3.io/p2p/go-p2p-testing v0.0.2
+)
+
+go 1.15

interface.go

+/*
+Package namesys implements resolvers and publishers for the DMS3
+naming system (DMS3NS).
+
+The core of DMS3 is an immutable, content-addressable Merkle graph.
+That works well for many use cases, but doesn't allow you to answer
+questions like "what is Alice's current homepage?".  The mutable name
+system allows Alice to publish information like:
+
+  The current homepage for alice.example.com is
+  /dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj
+
+or:
+
+  The current homepage for node
+  QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy
+  is
+  /dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj
+
+The mutable name system also allows users to resolve those references
+to find the immutable DMS3 object currently referenced by a given
+mutable name.
+
+For command-line bindings to this functionality, see:
+
+  dms3 name
+  dms3 dns
+  dms3 resolve
+*/
+package namesys
+
+import (
+	"errors"
+	"time"
+
+	context "context"
+
+	path "gitlab.dms3.io/dms3/go-path"
+	opts "gitlab.dms3.io/dms3/interface-go-dms3-core/options/namesys"
+	ci "gitlab.dms3.io/p2p/go-p2p-core/crypto"
+)
+
+// ErrResolveFailed signals an error when attempting to resolve.
+var ErrResolveFailed = errors.New("could not resolve name")
+
+// ErrResolveRecursion signals a recursion-depth limit.
+var ErrResolveRecursion = errors.New(
+	"could not resolve name (recursion limit exceeded)")
+
+// ErrPublishFailed signals an error when attempting to publish.
+var ErrPublishFailed = errors.New("could not publish name")
+
+// NameSystem represents a cohesive name publishing and resolving system.
+//
+// Publishing a name is the process of establishing a mapping, a key-value
+// pair, according to naming rules and databases.
+//
+// Resolving a name is the process of looking up the value associated with the
+// key (name).
+type NameSystem interface {
+	Resolver
+	Publisher
+}
+
+// Result is the return type for Resolver.ResolveAsync.
+type Result struct {
+	Path path.Path
+	Err  error
+}
+
+// Resolver is an object capable of resolving names.
+type Resolver interface {
+
+	// Resolve performs a recursive lookup, returning the dereferenced
+	// path.  For example, if dms3.io has a DNS TXT record pointing to
+	//   /dms3ns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy
+	// and there is a DHT DMS3NS entry for
+	//   QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy
+	//   -> /dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj
+	// then
+	//   Resolve(ctx, "/dms3ns/dms3.io")
+	// will resolve both names, returning
+	//   /dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj
+	//
+	// There is a default depth-limit to avoid infinite recursion.  Most
+	// users will be fine with this default limit, but if you need to
+	// adjust the limit you can specify it as an option.
+	Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (value path.Path, err error)
+
+	// ResolveAsync performs recursive name lookup, like Resolve, but it returns
+	// entries as they are discovered in the DHT. Each returned result is guaranteed
+	// to be "better" (which usually means newer) than the previous one.
+	ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result
+}
+
+// Publisher is an object capable of publishing particular names.
+type Publisher interface {
+
+	// Publish establishes a name-value mapping.
+	// TODO make this not PrivKey specific.
+	Publish(ctx context.Context, name ci.PrivKey, value path.Path) error
+
+	// TODO: to be replaced by a more generic 'PublishWithValidity' type
+	// call once the records spec is implemented
+	PublishWithEOL(ctx context.Context, name ci.PrivKey, value path.Path, eol time.Time) error
+}

namesys.go

+// Package namesys defines Resolver and Publisher interfaces for DMS3NS paths,
+// that is, DMS3 paths in the form of /dms3ns/<name_to_be_resolved>. A "resolved"
+// DMS3NS path becomes an /dms3/<cid> path.
+//
+// Traditionally, these paths would be in the form of /dms3ns/peer_id, which
+// references an DMS3NS record in a distributed ValueStore (usually the DMS3
+// DHT).
+//
+// Additionally, the /dms3ns/ namespace can also be used with domain names that
+// use DNSLink (/dms3ns/<dnslink_name>, https://docs.dms3.io/concepts/dnslink/)
+//
+// The package provides implementations for all three resolvers.
+package namesys
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+	dns "github.com/miekg/dns"
+	cid "gitlab.dms3.io/dms3/go-cid"
+	ds "gitlab.dms3.io/dms3/go-datastore"
+	dssync "gitlab.dms3.io/dms3/go-datastore/sync"
+	path "gitlab.dms3.io/dms3/go-path"
+	opts "gitlab.dms3.io/dms3/interface-go-dms3-core/options/namesys"
+	madns "gitlab.dms3.io/mf/go-multiaddr-dns"
+	ci "gitlab.dms3.io/p2p/go-p2p-core/crypto"
+	peer "gitlab.dms3.io/p2p/go-p2p-core/peer"
+	routing "gitlab.dms3.io/p2p/go-p2p-core/routing"
+)
+
+// mpns (a multi-protocol NameSystem) implements generic DMS3 naming.
+//
+// Uses several Resolvers:
+// (a) DMS3 routing naming: SFS-like PKI names.
+// (b) dns domains: resolves using links in DNS TXT records
+//
+// It can only publish to: (a) DMS3 routing naming.
+//
+type mpns struct {
+	ds ds.Datastore
+
+	dnsResolver, dms3nsResolver resolver
+	dms3nsPublisher             Publisher
+
+	staticMap map[string]path.Path
+	cache     *lru.Cache
+}
+
+type Option func(*mpns) error
+
+// WithCache is an option that instructs the name system to use a (LRU) cache of the given size.
+func WithCache(size int) Option {
+	return func(ns *mpns) error {
+		if size <= 0 {
+			return fmt.Errorf("invalid cache size %d; must be > 0", size)
+		}
+
+		cache, err := lru.New(size)
+		if err != nil {
+			return err
+		}
+
+		ns.cache = cache
+		return nil
+	}
+}
+
+// WithDNSResolver is an option that supplies a custom DNS resolver to use instead of the system
+// default.
+func WithDNSResolver(rslv madns.BasicResolver) Option {
+	return func(ns *mpns) error {
+		ns.dnsResolver = NewDNSResolver(rslv.LookupTXT)
+		return nil
+	}
+}
+
+// WithDatastore is an option that supplies a datastore to use instead of an in-memory map datastore. The datastore is used to store published DMS3NS records and make them available for querying.
+func WithDatastore(ds ds.Datastore) Option {
+	return func(ns *mpns) error {
+		ns.ds = ds
+		return nil
+	}
+}
+
+// NewNameSystem will construct the DMS3 naming system based on Routing
+func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) {
+	var staticMap map[string]path.Path
+
+	// Prewarm namesys cache with static records for deterministic tests and debugging.
+	// Useful for testing things like DNSLink without real DNS lookup.
+	// Example:
+	// DMS3_NS_MAP="dnslink-test.example.com:/dms3/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am"
+	if list := os.Getenv("DMS3_NS_MAP"); list != "" {
+		staticMap = make(map[string]path.Path)
+		for _, pair := range strings.Split(list, ",") {
+			mapping := strings.SplitN(pair, ":", 2)
+			key := mapping[0]
+			value := path.FromString(mapping[1])
+			staticMap[key] = value
+		}
+	}
+
+	ns := &mpns{
+		staticMap: staticMap,
+	}
+
+	for _, opt := range opts {
+		err := opt(ns)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if ns.ds == nil {
+		ns.ds = dssync.MutexWrap(ds.NewMapDatastore())
+	}
+
+	if ns.dnsResolver == nil {
+		ns.dnsResolver = NewDNSResolver(madns.DefaultResolver.LookupTXT)
+	}
+
+	ns.dms3nsResolver = NewDms3NsResolver(r)
+	ns.dms3nsPublisher = NewDms3NsPublisher(r, ns.ds)
+
+	return ns, nil
+}
+
+// DefaultResolverCacheTTL defines max ttl of a record placed in namesys cache.
+const DefaultResolverCacheTTL = time.Minute
+
+// Resolve implements Resolver.
+func (ns *mpns) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) {
+	if strings.HasPrefix(name, "/dms3/") {
+		return path.ParsePath(name)
+	}
+
+	if !strings.HasPrefix(name, "/") {
+		return path.ParsePath("/dms3/" + name)
+	}
+
+	return resolve(ctx, ns, name, opts.ProcessOpts(options))
+}
+
+func (ns *mpns) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result {
+	if strings.HasPrefix(name, "/dms3/") {
+		p, err := path.ParsePath(name)
+		res := make(chan Result, 1)
+		res <- Result{p, err}
+		close(res)
+		return res
+	}
+
+	if !strings.HasPrefix(name, "/") {
+		p, err := path.ParsePath("/dms3/" + name)
+		res := make(chan Result, 1)
+		res <- Result{p, err}
+		close(res)
+		return res
+	}
+
+	return resolveAsync(ctx, ns, name, opts.ProcessOpts(options))
+}
+
+// resolveOnce implements resolver.
+func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
+	out := make(chan onceResult, 1)
+
+	if !strings.HasPrefix(name, dms3nsPrefix) {
+		name = dms3nsPrefix + name
+	}
+	segments := strings.SplitN(name, "/", 4)
+	if len(segments) < 3 || segments[0] != "" {
+		log.Debugf("invalid name syntax for %s", name)
+		out <- onceResult{err: ErrResolveFailed}
+		close(out)
+		return out
+	}
+
+	key := segments[2]
+
+	// Resolver selection:
+	// 1. if it is a PeerID/CID/multihash resolve through "dms3ns".
+	// 2. if it is a domain name, resolve through "dns"
+
+	var res resolver
+	dms3nsKey, err := peer.Decode(key)
+
+	// CIDs in DMS3NS are expected to have p2p-key multicodec
+	// We ease the transition by returning a more meaningful error with a valid CID
+	if err != nil && err.Error() == "can't convert CID of type protobuf to a peer ID" {
+		dms3nsCid, cidErr := cid.Decode(key)
+		if cidErr == nil && dms3nsCid.Version() == 1 && dms3nsCid.Type() != cid.P2pKey {
+			fixedCid := cid.NewCidV1(cid.P2pKey, dms3nsCid.Hash()).String()
+			codecErr := fmt.Errorf("peer ID represented as CIDv1 require p2p-key multicodec: retry with /dms3ns/%s", fixedCid)
+			log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", key, codecErr)
+			out <- onceResult{err: codecErr}
+			close(out)
+			return out
+		}
+	}
+
+	cacheKey := key
+	if err == nil {
+		cacheKey = string(dms3nsKey)
+	}
+
+	if p, ok := ns.cacheGet(cacheKey); ok {
+		var err error
+		if len(segments) > 3 {
+			p, err = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3])
+		}
+
+		out <- onceResult{value: p, err: err}
+		close(out)
+		return out
+	}
+
+	if err == nil {
+		res = ns.dms3nsResolver
+	} else if _, ok := dns.IsDomainName(key); ok {
+		res = ns.dnsResolver
+	} else {
+		out <- onceResult{err: fmt.Errorf("invalid DMS3NS root: %q", key)}
+		close(out)
+		return out
+	}
+
+	resCh := res.resolveOnceAsync(ctx, key, options)
+	var best onceResult
+	go func() {
+		defer close(out)
+		for {
+			select {
+			case res, ok := <-resCh:
+				if !ok {
+					if best != (onceResult{}) {
+						ns.cacheSet(cacheKey, best.value, best.ttl)
+					}
+					return
+				}
+				if res.err == nil {
+					best = res
+				}
+				p := res.value
+				err := res.err
+				ttl := res.ttl
+
+				// Attach rest of the path
+				if len(segments) > 3 {
+					p, err = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3])
+				}
+
+				emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl, err: err})
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return out
+}
+
+func emitOnceResult(ctx context.Context, outCh chan<- onceResult, r onceResult) {
+	select {
+	case outCh <- r:
+	case <-ctx.Done():
+	}
+}
+
+// Publish implements Publisher
+func (ns *mpns) Publish(ctx context.Context, name ci.PrivKey, value path.Path) error {
+	return ns.PublishWithEOL(ctx, name, value, time.Now().Add(DefaultRecordEOL))
+}
+
+func (ns *mpns) PublishWithEOL(ctx context.Context, name ci.PrivKey, value path.Path, eol time.Time) error {
+	id, err := peer.IDFromPrivateKey(name)
+	if err != nil {
+		return err
+	}
+	if err := ns.dms3nsPublisher.PublishWithEOL(ctx, name, value, eol); err != nil {
+		// Invalidate the cache. Publishing may _partially_ succeed but
+		// still return an error.
+		ns.cacheInvalidate(string(id))
+		return err
+	}
+	ttl := DefaultResolverCacheTTL
+	if setTTL, ok := checkCtxTTL(ctx); ok {
+		ttl = setTTL
+	}
+	if ttEol := time.Until(eol); ttEol < ttl {
+		ttl = ttEol
+	}
+	ns.cacheSet(string(id), value, ttl)
+	return nil
+}

namesys_test.go

+package namesys
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	ds "gitlab.dms3.io/dms3/go-datastore"
+	dssync "gitlab.dms3.io/dms3/go-datastore/sync"
+	offroute "gitlab.dms3.io/dms3/go-dms3-routing/offline"
+	dms3ns "gitlab.dms3.io/dms3/go-dms3ns"
+	path "gitlab.dms3.io/dms3/go-path"
+	opts "gitlab.dms3.io/dms3/interface-go-dms3-core/options/namesys"
+	ci "gitlab.dms3.io/p2p/go-p2p-core/crypto"
+	peer "gitlab.dms3.io/p2p/go-p2p-core/peer"
+	pstoremem "gitlab.dms3.io/p2p/go-p2p-peerstore/pstoremem"
+	record "gitlab.dms3.io/p2p/go-p2p-record"
+)
+
+type mockResolver struct {
+	entries map[string]string
+}
+
+func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expError error) {
+	t.Helper()
+	p, err := resolver.Resolve(context.Background(), name, opts.Depth(depth))
+	if !errors.Is(err, expError) {
+		t.Fatal(fmt.Errorf(
+			"expected %s with a depth of %d to have a '%s' error, but got '%s'",
+			name, depth, expError, err))
+	}
+	if p.String() != expected {
+		t.Fatal(fmt.Errorf(
+			"%s with depth %d resolved to %s != %s",
+			name, depth, p.String(), expected))
+	}
+}
+
+func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
+	p, err := path.ParsePath(r.entries[name])
+	out := make(chan onceResult, 1)
+	out <- onceResult{value: p, err: err}
+	close(out)
+	return out
+}
+
+func mockResolverOne() *mockResolver {
+	return &mockResolver{
+		entries: map[string]string{
+			"QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy":              "/dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj",
+			"QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n":              "/dms3ns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy",
+			"QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD":              "/dms3ns/dms3.io",
+			"QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei":              "/dms3/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act",
+			"12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5":        "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // ed25519+identity multihash
+			"bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm": "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // cidv1 in base32 with p2p-key multicodec
+		},
+	}
+}
+
+func mockResolverTwo() *mockResolver {
+	return &mockResolver{
+		entries: map[string]string{
+			"dms3.io": "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n",
+		},
+	}
+}
+
+func TestNamesysResolution(t *testing.T) {
+	r := &mpns{
+		dms3nsResolver: mockResolverOne(),
+		dnsResolver:    mockResolverTwo(),
+	}
+
+	testResolution(t, r, "Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", opts.DefaultDepthLimit, "/dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+	testResolution(t, r, "/dms3ns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", opts.DefaultDepthLimit, "/dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+	testResolution(t, r, "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", opts.DefaultDepthLimit, "/dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+	testResolution(t, r, "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/dms3ns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion)
+	testResolution(t, r, "/dms3ns/dms3.io", opts.DefaultDepthLimit, "/dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+	testResolution(t, r, "/dms3ns/dms3.io", 1, "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion)
+	testResolution(t, r, "/dms3ns/dms3.io", 2, "/dms3ns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion)
+	testResolution(t, r, "/dms3ns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", opts.DefaultDepthLimit, "/dms3/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil)
+	testResolution(t, r, "/dms3ns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/dms3ns/dms3.io", ErrResolveRecursion)
+	testResolution(t, r, "/dms3ns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion)
+	testResolution(t, r, "/dms3ns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/dms3ns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion)
+	testResolution(t, r, "/dms3ns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion)
+	testResolution(t, r, "/dms3ns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/dms3ns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion)
+}
+
+func TestPublishWithCache0(t *testing.T) {
+	dst := dssync.MutexWrap(ds.NewMapDatastore())
+	priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ps := pstoremem.NewPeerstore()
+	pid, err := peer.IDFromPrivateKey(priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ps.AddPrivKey(pid, priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{
+		"dms3ns": dms3ns.Validator{KeyBook: ps},
+		"pk":     record.PublicKeyValidator{},
+	})
+
+	nsys, err := NewNameSystem(routing, WithDatastore(dst))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// CID is arbitrary.
+	p, err := path.ParsePath("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = nsys.Publish(context.Background(), priv, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestPublishWithTTL(t *testing.T) {
+	dst := dssync.MutexWrap(ds.NewMapDatastore())
+	priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ps := pstoremem.NewPeerstore()
+	pid, err := peer.IDFromPrivateKey(priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ps.AddPrivKey(pid, priv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{
+		"dms3ns": dms3ns.Validator{KeyBook: ps},
+		"pk":     record.PublicKeyValidator{},
+	})
+
+	nsys, err := NewNameSystem(routing, WithDatastore(dst), WithCache(128))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// CID is arbitrary.
+	p, err := path.ParsePath("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ttl := 1 * time.Second
+	eol := time.Now().Add(2 * time.Second)
+
+	ctx := ContextWithTTL(context.Background(), ttl)
+	err = nsys.Publish(ctx, priv, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ientry, ok := nsys.(*mpns).cache.Get(string(pid))
+	if !ok {
+		t.Fatal("cache get failed")
+	}
+	entry, ok := ientry.(cacheEntry)
+	if !ok {
+		t.Fatal("bad cache item returned")
+	}
+	if entry.eol.Sub(eol) > 10*time.Millisecond {
+		t.Fatalf("bad cache ttl: expected %s, got %s", eol, entry.eol)
+	}
+}

publisher.go

+package namesys
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	proto "github.com/gogo/protobuf/proto"
+	base32 "github.com/whyrusleeping/base32"
+	ds "gitlab.dms3.io/dms3/go-datastore"
+	dsquery "gitlab.dms3.io/dms3/go-datastore/query"
+	dms3ns "gitlab.dms3.io/dms3/go-dms3ns"
+	pb "gitlab.dms3.io/dms3/go-dms3ns/pb"
+	path "gitlab.dms3.io/dms3/go-path"
+	ci "gitlab.dms3.io/p2p/go-p2p-core/crypto"
+	peer "gitlab.dms3.io/p2p/go-p2p-core/peer"
+	routing "gitlab.dms3.io/p2p/go-p2p-core/routing"
+)
+
+const dms3nsPrefix = "/dms3ns/"
+
+// DefaultRecordEOL specifies the time that the network will cache DMS3NS
+// records after being publihsed. Records should be re-published before this
+// interval expires.
+const DefaultRecordEOL = 24 * time.Hour
+
+// Dms3NsPublisher is capable of publishing and resolving names to the DMS3
+// routing system.
+type Dms3NsPublisher struct {
+	routing routing.ValueStore
+	ds      ds.Datastore
+
+	// Used to ensure we assign DMS3NS records *sequential* sequence numbers.
+	mu sync.Mutex
+}
+
+// NewDms3NsPublisher constructs a publisher for the DMS3 Routing name system.
+func NewDms3NsPublisher(route routing.ValueStore, ds ds.Datastore) *Dms3NsPublisher {
+	if ds == nil {
+		panic("nil datastore")
+	}
+	return &Dms3NsPublisher{routing: route, ds: ds}
+}
+
+// Publish implements Publisher. Accepts a keypair and a value,
+// and publishes it out to the routing system
+func (p *Dms3NsPublisher) Publish(ctx context.Context, k ci.PrivKey, value path.Path) error {
+	log.Debugf("Publish %s", value)
+	return p.PublishWithEOL(ctx, k, value, time.Now().Add(DefaultRecordEOL))
+}
+
+// Dms3NsDsKey returns a datastore key given an DMS3NS identifier (peer
+// ID). Defines the storage key for DMS3NS records in the local datastore.
+func Dms3NsDsKey(id peer.ID) ds.Key {
+	return ds.NewKey("/dms3ns/" + base32.RawStdEncoding.EncodeToString([]byte(id)))
+}
+
+// ListPublished returns the latest DMS3NS records published by this node and
+// their expiration times.
+//
+// This method will not search the routing system for records published by other
+// nodes.
+func (p *Dms3NsPublisher) ListPublished(ctx context.Context) (map[peer.ID]*pb.Dms3NsEntry, error) {
+	query, err := p.ds.Query(dsquery.Query{
+		Prefix: dms3nsPrefix,
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer query.Close()
+
+	records := make(map[peer.ID]*pb.Dms3NsEntry)
+	for {
+		select {
+		case result, ok := <-query.Next():
+			if !ok {
+				return records, nil
+			}
+			if result.Error != nil {
+				return nil, result.Error
+			}
+			e := new(pb.Dms3NsEntry)
+			if err := proto.Unmarshal(result.Value, e); err != nil {
+				// Might as well return what we can.
+				log.Error("found an invalid DMS3NS entry:", err)
+				continue
+			}
+			if !strings.HasPrefix(result.Key, dms3nsPrefix) {
+				log.Errorf("datastore query for keys with prefix %s returned a key: %s", dms3nsPrefix, result.Key)
+				continue
+			}
+			k := result.Key[len(dms3nsPrefix):]
+			pid, err := base32.RawStdEncoding.DecodeString(k)
+			if err != nil {
+				log.Errorf("dms3ns ds key invalid: %s", result.Key)
+				continue
+			}
+			records[peer.ID(pid)] = e
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+}
+
+// GetPublished returns the record this node has published corresponding to the
+// given peer ID.
+//
+// If `checkRouting` is true and we have no existing record, this method will
+// check the routing system for any existing records.
+func (p *Dms3NsPublisher) GetPublished(ctx context.Context, id peer.ID, checkRouting bool) (*pb.Dms3NsEntry, error) {
+	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+
+	value, err := p.ds.Get(Dms3NsDsKey(id))
+	switch err {
+	case nil:
+	case ds.ErrNotFound:
+		if !checkRouting {
+			return nil, nil
+		}
+		dms3nskey := dms3ns.RecordKey(id)
+		value, err = p.routing.GetValue(ctx, dms3nskey)
+		if err != nil {
+			// Not found or other network issue. Can't really do
+			// anything about this case.
+			if err != routing.ErrNotFound {
+				log.Debugf("error when determining the last published DMS3NS record for %s: %s", id, err)
+			}
+
+			return nil, nil
+		}
+	default:
+		return nil, err
+	}
+	e := new(pb.Dms3NsEntry)
+	if err := proto.Unmarshal(value, e); err != nil {
+		return nil, err
+	}
+	return e, nil
+}
+
+func (p *Dms3NsPublisher) updateRecord(ctx context.Context, k ci.PrivKey, value path.Path, eol time.Time) (*pb.Dms3NsEntry, error) {
+	id, err := peer.IDFromPrivateKey(k)
+	if err != nil {
+		return nil, err
+	}
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	// get previous records sequence number
+	rec, err := p.GetPublished(ctx, id, true)
+	if err != nil {
+		return nil, err
+	}
+
+	seqno := rec.GetSequence() // returns 0 if rec is nil
+	if rec != nil && value != path.Path(rec.GetValue()) {
+		// Don't bother incrementing the sequence number unless the
+		// value changes.
+		seqno++
+	}
+
+	// Create record
+	entry, err := dms3ns.Create(k, []byte(value), seqno, eol)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set the TTL
+	// TODO: Make this less hacky.
+	ttl, ok := checkCtxTTL(ctx)
+	if ok {
+		entry.Ttl = proto.Uint64(uint64(ttl.Nanoseconds()))
+	}
+
+	data, err := proto.Marshal(entry)
+	if err != nil {
+		return nil, err
+	}
+
+	// Put the new record.
+	key := Dms3NsDsKey(id)
+	if err := p.ds.Put(key, data); err != nil {
+		return nil, err
+	}
+	if err := p.ds.Sync(key); err != nil {
+		return nil, err
+	}
+	return entry, nil
+}
+
+// PublishWithEOL is a temporary stand in for the dms3ns records implementation
+// see here for more details: https://gitlab.dms3.io/dms3/specs/tree/master/records
+func (p *Dms3NsPublisher) PublishWithEOL(ctx context.Context, k ci.PrivKey, value path.Path, eol time.Time) error {
+	record, err := p.updateRecord(ctx, k, value, eol)
+	if err != nil {
+		return err
+	}
+
+	return PutRecordToRouting(ctx, p.routing, k.GetPublic(), record)
+}
+
+// setting the TTL on published records is an experimental feature.
+// as such, i'm using the context to wire it through to avoid changing too
+// much code along the way.
+func checkCtxTTL(ctx context.Context) (time.Duration, bool) {
+	v := ctx.Value(ttlContextKey)
+	if v == nil {
+		return 0, false
+	}
+
+	d, ok := v.(time.Duration)
+	return d, ok
+}
+
+// PutRecordToRouting publishes the given entry using the provided ValueStore,
+// keyed on the ID associated with the provided public key. The public key is
+// also made available to the routing system so that entries can be verified.
+func PutRecordToRouting(ctx context.Context, r routing.ValueStore, k ci.PubKey, entry *pb.Dms3NsEntry) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	errs := make(chan error, 2) // At most two errors (DMS3NS, and public key)
+
+	if err := dms3ns.EmbedPublicKey(k, entry); err != nil {
+		return err
+	}
+
+	id, err := peer.IDFromPublicKey(k)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		errs <- PublishEntry(ctx, r, dms3ns.RecordKey(id), entry)
+	}()
+
+	// Publish the public key if a public key cannot be extracted from the ID
+	// TODO: once v0.4.16 is widespread enough, we can stop doing this
+	// and at that point we can even deprecate the /pk/ namespace in the dht
+	//
+	// NOTE: This check actually checks if the public key has been embedded
+	// in the DMS3NS entry. This check is sufficient because we embed the
+	// public key in the DMS3NS entry if it can't be extracted from the ID.
+	if entry.PubKey != nil {
+		go func() {
+			errs <- PublishPublicKey(ctx, r, PkKeyForID(id), k)
+		}()
+
+		if err := waitOnErrChan(ctx, errs); err != nil {
+			return err
+		}
+	}
+
+	return waitOnErrChan(ctx, errs)
+}
+
+func waitOnErrChan(ctx context.Context, errs chan error) error {
+	select {
+	case err := <-errs:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// PublishPublicKey stores the given public key in the ValueStore with the
+// given key.
+func PublishPublicKey(ctx context.Context, r routing.ValueStore, k string, pubk ci.PubKey) error {
+	log.Debugf("Storing pubkey at: %s", k)
+	pkbytes, err := pubk.Bytes()
+	if err != nil {
+		return err
+	}
+
+	// Store associated public key
+	return r.PutValue(ctx, k, pkbytes)
+}
+
+// PublishEntry stores the given Dms3NsEntry in the ValueStore with the given
+// dms3nskey.
+func PublishEntry(ctx context.Context, r routing.ValueStore, dms3nskey string, rec *pb.Dms3NsEntry) error {
+	data, err := proto.Marshal(rec)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("Storing dms3ns entry at: %s", dms3nskey)
+	// Store dms3ns entry at "/dms3ns/"+h(pubkey)
+	return r.PutValue(ctx, dms3nskey, data)
+}
+
+// PkKeyForID returns the public key routing key for the given peer ID.
+func PkKeyForID(id peer.ID) string {
+	return "/pk/" + string(id)
+}
+
+// contextKey is a private comparable type used to hold value keys in contexts
+type contextKey string
+
+var ttlContextKey contextKey = "dms3ns-publish-ttl"
+
+// ContextWithTTL returns a copy of the parent context with an added value representing the TTL
+func ContextWithTTL(ctx context.Context, ttl time.Duration) context.Context {
+	return context.WithValue(context.Background(), ttlContextKey, ttl)
+}

publisher_test.go

+package namesys
+
+import (
+	"context"
+	"crypto/rand"
+	"testing"
+	"time"
+
+	"gitlab.dms3.io/dms3/go-path"
+
+	ds "gitlab.dms3.io/dms3/go-datastore"
+	dssync "gitlab.dms3.io/dms3/go-datastore/sync"
+	dshelp "gitlab.dms3.io/dms3/go-dms3-ds-help"
+	mockrouting "gitlab.dms3.io/dms3/go-dms3-routing/mock"
+	dms3ns "gitlab.dms3.io/dms3/go-dms3ns"
+	ma "gitlab.dms3.io/mf/go-multiaddr"
+	ci "gitlab.dms3.io/p2p/go-p2p-core/crypto"
+	peer "gitlab.dms3.io/p2p/go-p2p-core/peer"
+	testutil "gitlab.dms3.io/p2p/go-p2p-testing/net"
+)
+
+type identity struct {
+	testutil.PeerNetParams
+}
+
+func (p *identity) ID() peer.ID {
+	return p.PeerNetParams.ID
+}
+
+func (p *identity) Address() ma.Multiaddr {
+	return p.Addr
+}
+
+func (p *identity) PrivateKey() ci.PrivKey {
+	return p.PrivKey
+}
+
+func (p *identity) PublicKey() ci.PubKey {
+	return p.PubKey
+}
+
+func testNamekeyPublisher(t *testing.T, keyType int, expectedErr error, expectedExistence bool) {
+	// Context
+	ctx := context.Background()
+
+	// Private key
+	privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ID
+	id, err := peer.IDFromPublicKey(pubKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Value
+	value := []byte("dms3/TESTING")
+
+	// Seqnum
+	seqnum := uint64(0)
+
+	// Eol
+	eol := time.Now().Add(24 * time.Hour)
+
+	// Routing value store
+	p := testutil.PeerNetParams{
+		ID:      id,
+		PrivKey: privKey,
+		PubKey:  pubKey,
+		Addr:    testutil.ZeroLocalTCPAddress,
+	}
+
+	dstore := dssync.MutexWrap(ds.NewMapDatastore())
+	serv := mockrouting.NewServer()
+	r := serv.ClientWithDatastore(context.Background(), &identity{p}, dstore)
+
+	entry, err := dms3ns.Create(privKey, value, seqnum, eol)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = PutRecordToRouting(ctx, r, pubKey, entry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check for namekey existence in value store
+	namekey := PkKeyForID(id)
+	_, err = r.GetValue(ctx, namekey)
+	if err != expectedErr {
+		t.Fatal(err)
+	}
+
+	// Also check datastore for completeness
+	key := dshelp.NewKeyFromBinary([]byte(namekey))
+	exists, err := dstore.Has(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if exists != expectedExistence {
+		t.Fatal("Unexpected key existence in datastore")
+	}
+}
+
+func TestRSAPublisher(t *testing.T) {
+	testNamekeyPublisher(t, ci.RSA, nil, true)
+}
+
+func TestEd22519Publisher(t *testing.T) {
+	testNamekeyPublisher(t, ci.Ed25519, ds.ErrNotFound, false)
+}
+
+func TestAsyncDS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	rt := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t))
+	ds := &checkSyncDS{
+		Datastore: ds.NewMapDatastore(),
+		syncKeys:  make(map[ds.Key]struct{}),
+	}
+	publisher := NewDms3NsPublisher(rt, ds)
+
+	dms3nsFakeID := testutil.RandIdentityOrFatal(t)
+	dms3nsVal, err := path.ParsePath("/dms3ns/foo.bar")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := publisher.Publish(ctx, dms3nsFakeID.PrivateKey(), dms3nsVal); err != nil {
+		t.Fatal(err)
+	}
+
+	dms3nsKey := Dms3NsDsKey(dms3nsFakeID.ID())
+
+	for k := range ds.syncKeys {
+		if k.IsAncestorOf(dms3nsKey) || k.Equal(dms3nsKey) {
+			return
+		}
+	}
+
+	t.Fatal("dms3ns key not synced")
+}
+
+type checkSyncDS struct {
+	ds.Datastore
+	syncKeys map[ds.Key]struct{}
+}
+
+func (d *checkSyncDS) Sync(prefix ds.Key) error {
+	d.syncKeys[prefix] = struct{}{}
+	return d.Datastore.Sync(prefix)
+}

republisher/repub.go

+// Package republisher provides a utility to automatically re-publish DMS3NS
+// records related to the keys in a Keystore.
+package republisher
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	keystore "gitlab.dms3.io/dms3/go-dms3-keystore"
+	namesys "gitlab.dms3.io/dms3/go-namesys"
+	path "gitlab.dms3.io/dms3/go-path"
+
+	proto "github.com/gogo/protobuf/proto"
+	goprocess "github.com/jbenet/goprocess"
+	gpctx "github.com/jbenet/goprocess/context"
+	ds "gitlab.dms3.io/dms3/go-datastore"
+	dms3ns "gitlab.dms3.io/dms3/go-dms3ns"
+	pb "gitlab.dms3.io/dms3/go-dms3ns/pb"
+	logging "gitlab.dms3.io/dms3/go-log"
+	ic "gitlab.dms3.io/p2p/go-p2p-core/crypto"
+	peer "gitlab.dms3.io/p2p/go-p2p-core/peer"
+)
+
+var errNoEntry = errors.New("no previous entry")
+
+var log = logging.Logger("dms3ns-repub")
+
+// DefaultRebroadcastInterval is the default interval at which we rebroadcast DMS3NS records
+var DefaultRebroadcastInterval = time.Hour * 4
+
+// InitialRebroadcastDelay is the delay before first broadcasting DMS3NS records on start
+var InitialRebroadcastDelay = time.Minute * 1
+
+// FailureRetryInterval is the interval at which we retry DMS3NS records broadcasts (when they fail)
+var FailureRetryInterval = time.Minute * 5
+
+// DefaultRecordLifetime is the default lifetime for DMS3NS records
+const DefaultRecordLifetime = time.Hour * 24
+
+// Republisher facilitates the regular publishing of all the DMS3NS records
+// associated to keys in a Keystore.
+type Republisher struct {
+	ns   namesys.Publisher
+	ds   ds.Datastore
+	self ic.PrivKey
+	ks   keystore.Keystore
+
+	Interval time.Duration
+
+	// how long records that are republished should be valid for
+	RecordLifetime time.Duration
+}
+
+// NewRepublisher creates a new Republisher
+func NewRepublisher(ns namesys.Publisher, ds ds.Datastore, self ic.PrivKey, ks keystore.Keystore) *Republisher {
+	return &Republisher{
+		ns:             ns,
+		ds:             ds,
+		self:           self,
+		ks:             ks,
+		Interval:       DefaultRebroadcastInterval,
+		RecordLifetime: DefaultRecordLifetime,
+	}
+}
+
+// Run starts the republisher facility. It can be stopped by stopping the
+// provided proc.
+func (rp *Republisher) Run(proc goprocess.Process) {
+	timer := time.NewTimer(InitialRebroadcastDelay)
+	defer timer.Stop()
+	if rp.Interval < InitialRebroadcastDelay {
+		timer.Reset(rp.Interval)
+	}
+
+	for {
+		select {
+		case <-timer.C:
+			timer.Reset(rp.Interval)
+			err := rp.republishEntries(proc)
+			if err != nil {
+				log.Info("republisher failed to republish: ", err)
+				if FailureRetryInterval < rp.Interval {
+					timer.Reset(FailureRetryInterval)
+				}
+			}
+		case <-proc.Closing():
+			return
+		}
+	}
+}
+
+func (rp *Republisher) republishEntries(p goprocess.Process) error {
+	ctx, cancel := context.WithCancel(gpctx.OnClosingContext(p))
+	defer cancel()
+
+	// TODO: Use rp.dms3ns.ListPublished(). We can't currently *do* that
+	// because:
+	// 1. There's no way to get keys from the keystore by ID.
+	// 2. We don't actually have access to the DMS3NS publisher.
+	err := rp.republishEntry(ctx, rp.self)
+	if err != nil {
+		return err
+	}
+
+	if rp.ks != nil {
+		keyNames, err := rp.ks.List()
+		if err != nil {
+			return err
+		}
+		for _, name := range keyNames {
+			priv, err := rp.ks.Get(name)
+			if err != nil {
+				return err
+			}
+			err = rp.republishEntry(ctx, priv)
+			if err != nil {
+				return err
+			}
+
+		}
+	}
+
+	return nil
+}
+
+func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) error {
+	id, err := peer.IDFromPrivateKey(priv)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("republishing dms3ns entry for %s", id)
+
+	// Look for it locally only
+	e, err := rp.getLastDMS3NSEntry(id)
+	if err != nil {
+		if err == errNoEntry {
+			return nil
+		}
+		return err
+	}
+
+	p := path.Path(e.GetValue())
+	prevEol, err := dms3ns.GetEOL(e)
+	if err != nil {
+		return err
+	}
+
+	// update record with same sequence number
+	eol := time.Now().Add(rp.RecordLifetime)
+	if prevEol.After(eol) {
+		eol = prevEol
+	}
+	return rp.ns.PublishWithEOL(ctx, priv, p, eol)
+}
+
+func (rp *Republisher) getLastDMS3NSEntry(id peer.ID) (*pb.Dms3NsEntry, error) {
+	// Look for it locally only
+	val, err := rp.ds.Get(namesys.Dms3NsDsKey(id))
+	switch err {
+	case nil:
+	case ds.ErrNotFound:
+		return nil, errNoEntry
+	default:
+		return nil, err
+	}
+
+	e := new(pb.Dms3NsEntry)
+	if err := proto.Unmarshal(val, e); err != nil {
+		return nil, err
+	}
+	return e, nil
+}

republisher/repub_test.go

+package republisher_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+
+	goprocess "github.com/jbenet/goprocess"
+	p2p "gitlab.dms3.io/p2p/go-p2p"
+	ic "gitlab.dms3.io/p2p/go-p2p-core/crypto"
+	host "gitlab.dms3.io/p2p/go-p2p-core/host"
+	peer "gitlab.dms3.io/p2p/go-p2p-core/peer"
+	routing "gitlab.dms3.io/p2p/go-p2p-core/routing"
+	dht "gitlab.dms3.io/p2p/go-p2p-kad-dht"
+
+	ds "gitlab.dms3.io/dms3/go-datastore"
+	dssync "gitlab.dms3.io/dms3/go-datastore/sync"
+	"gitlab.dms3.io/dms3/go-dms3ns"
+	dms3ns_pb "gitlab.dms3.io/dms3/go-dms3ns/pb"
+	path "gitlab.dms3.io/dms3/go-path"
+
+	keystore "gitlab.dms3.io/dms3/go-dms3-keystore"
+	namesys "gitlab.dms3.io/dms3/go-namesys"
+	. "gitlab.dms3.io/dms3/go-namesys/republisher"
+)
+
+type mockNode struct {
+	h        host.Host
+	id       string
+	privKey  ic.PrivKey
+	store    ds.Batching
+	dht      *dht.Dms3DHT
+	keystore keystore.Keystore
+}
+
+func getMockNode(t *testing.T, ctx context.Context) *mockNode {
+	t.Helper()
+
+	dstore := dssync.MutexWrap(ds.NewMapDatastore())
+	var idht *dht.Dms3DHT
+	h, err := p2p.New(
+		ctx,
+		p2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"),
+		p2p.Routing(func(h host.Host) (routing.PeerRouting, error) {
+			rt, err := dht.New(ctx, h, dht.Mode(dht.ModeServer))
+			idht = rt
+			return rt, err
+		}),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return &mockNode{
+		h:        h,
+		id:       h.ID().Pretty(),
+		privKey:  h.Peerstore().PrivKey(h.ID()),
+		store:    dstore,
+		dht:      idht,
+		keystore: keystore.NewMemKeystore(),
+	}
+}
+
+func TestRepublish(t *testing.T) {
+	// set cache life to zero for testing low-period repubs
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var nsystems []namesys.NameSystem
+	var nodes []*mockNode
+	for i := 0; i < 10; i++ {
+		n := getMockNode(t, ctx)
+		ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		nsystems = append(nsystems, ns)
+		nodes = append(nodes, n)
+	}
+
+	pinfo := host.InfoFromHost(nodes[0].h)
+
+	for _, n := range nodes[1:] {
+		if err := n.h.Connect(ctx, *pinfo); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// have one node publish a record that is valid for 1 second
+	publisher := nodes[3]
+
+	p := path.FromString("/dms3/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid
+	rp := namesys.NewDms3NsPublisher(publisher.dht, publisher.store)
+	name := "/dms3ns/" + publisher.id
+
+	// Retry in case the record expires before we can fetch it. This can
+	// happen when running the test on a slow machine.
+	var expiration time.Time
+	timeout := time.Second
+	for {
+		expiration = time.Now().Add(time.Second)
+		err := rp.PublishWithEOL(ctx, publisher.privKey, p, expiration)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = verifyResolution(nsystems, name, p)
+		if err == nil {
+			break
+		}
+
+		if time.Now().After(expiration) {
+			timeout *= 2
+			continue
+		}
+		t.Fatal(err)
+	}
+
+	// Now wait a second, the records will be invalid and we should fail to resolve
+	time.Sleep(timeout)
+	if err := verifyResolutionFails(nsystems, name); err != nil {
+		t.Fatal(err)
+	}
+
+	// The republishers that are contained within the nodes have their timeout set
+	// to 12 hours. Instead of trying to tweak those, we're just going to pretend
+	// they don't exist and make our own.
+	repub := NewRepublisher(rp, publisher.store, publisher.privKey, publisher.keystore)
+	repub.Interval = time.Second
+	repub.RecordLifetime = time.Second * 5
+
+	proc := goprocess.Go(repub.Run)
+	defer proc.Close()
+
+	// now wait a couple seconds for it to fire
+	time.Sleep(time.Second * 2)
+
+	// we should be able to resolve them now
+	if err := verifyResolution(nsystems, name, p); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestLongEOLRepublish(t *testing.T) {
+	// set cache life to zero for testing low-period repubs
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var nsystems []namesys.NameSystem
+	var nodes []*mockNode
+	for i := 0; i < 10; i++ {
+		n := getMockNode(t, ctx)
+		ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		nsystems = append(nsystems, ns)
+		nodes = append(nodes, n)
+	}
+
+	pinfo := host.InfoFromHost(nodes[0].h)
+
+	for _, n := range nodes[1:] {
+		if err := n.h.Connect(ctx, *pinfo); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// have one node publish a record that is valid for 1 second
+	publisher := nodes[3]
+	p := path.FromString("/dms3/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid
+	rp := namesys.NewDms3NsPublisher(publisher.dht, publisher.store)
+	name := "/dms3ns/" + publisher.id
+
+	expiration := time.Now().Add(time.Hour)
+	err := rp.PublishWithEOL(ctx, publisher.privKey, p, expiration)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = verifyResolution(nsystems, name, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// The republishers that are contained within the nodes have their timeout set
+	// to 12 hours. Instead of trying to tweak those, we're just going to pretend
+	// they don't exist and make our own.
+	repub := NewRepublisher(rp, publisher.store, publisher.privKey, publisher.keystore)
+	repub.Interval = time.Millisecond * 500
+	repub.RecordLifetime = time.Second
+
+	proc := goprocess.Go(repub.Run)
+	defer proc.Close()
+
+	// now wait a couple seconds for it to fire a few times
+	time.Sleep(time.Second * 2)
+
+	err = verifyResolution(nsystems, name, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	entry, err := getLastDMS3NSEntry(publisher.store, publisher.h.ID())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	finalEol, err := dms3ns.GetEOL(entry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !finalEol.Equal(expiration) {
+		t.Fatal("expiration time modified")
+	}
+}
+
+func getLastDMS3NSEntry(dstore ds.Datastore, id peer.ID) (*dms3ns_pb.Dms3NsEntry, error) {
+	// Look for it locally only
+	val, err := dstore.Get(namesys.Dms3NsDsKey(id))
+	if err != nil {
+		return nil, err
+	}
+
+	e := new(dms3ns_pb.Dms3NsEntry)
+	if err := proto.Unmarshal(val, e); err != nil {
+		return nil, err
+	}
+	return e, nil
+}
+
+func verifyResolution(nsystems []namesys.NameSystem, key string, exp path.Path) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	for _, n := range nsystems {
+		val, err := n.Resolve(ctx, key)
+		if err != nil {
+			return err
+		}
+
+		if val != exp {
+			return errors.New("resolved wrong record")
+		}
+	}
+	return nil
+}
+
+func verifyResolutionFails(nsystems []namesys.NameSystem, key string) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	for _, n := range nsystems {
+		_, err := n.Resolve(ctx, key)
+		if err == nil {
+			return errors.New("expected resolution to fail")
+		}
+	}
+	return nil
+}

resolve/resolve.go

+package resolve
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"gitlab.dms3.io/dms3/go-path"
+
+	"gitlab.dms3.io/dms3/go-namesys"
+)
+
+// ErrNoNamesys is an explicit error for when an DMS3 node doesn't
+// (yet) have a name system
+var ErrNoNamesys = errors.New(
+	"core/resolve: no Namesys on Dms3Node - can't resolve dms3ns entry")
+
+// ResolveDMS3NS resolves /dms3ns paths
+func ResolveDMS3NS(ctx context.Context, nsys namesys.NameSystem, p path.Path) (path.Path, error) {
+	if strings.HasPrefix(p.String(), "/dms3ns/") {
+		// TODO(cryptix): we should be able to query the local cache for the path
+		if nsys == nil {
+			return "", ErrNoNamesys
+		}
+
+		seg := p.Segments()
+
+		if len(seg) < 2 || seg[1] == "" { // just "/<protocol/>" without further segments
+			err := fmt.Errorf("invalid path %q: dms3ns path missing DMS3NS ID", p)
+			return "", err
+		}
+
+		extensions := seg[2:]
+		resolvable, err := path.FromSegments("/", seg[0], seg[1])
+		if err != nil {
+			return "", err
+		}
+
+		respath, err := nsys.Resolve(ctx, resolvable.String())
+		if err != nil {
+			return "", err
+		}
+
+		segments := append(respath.Segments(), extensions...)
+		p, err = path.FromSegments("/", segments...)
+		if err != nil {
+			return "", err
+		}
+	}
+	return p, nil
+}

resolve_test.go

+package namesys
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	ds "gitlab.dms3.io/dms3/go-datastore"
+	dssync "gitlab.dms3.io/dms3/go-datastore/sync"
+	mockrouting "gitlab.dms3.io/dms3/go-dms3-routing/mock"
+	dms3ns "gitlab.dms3.io/dms3/go-dms3ns"
+	path "gitlab.dms3.io/dms3/go-path"
+	tnet "gitlab.dms3.io/p2p/go-p2p-testing/net"
+)
+
+func TestRoutingResolve(t *testing.T) {
+	dstore := dssync.MutexWrap(ds.NewMapDatastore())
+	serv := mockrouting.NewServer()
+	id := tnet.RandIdentityOrFatal(t)
+	d := serv.ClientWithDatastore(context.Background(), id, dstore)
+
+	resolver := NewDms3NsResolver(d)
+	publisher := NewDms3NsPublisher(d, dstore)
+
+	identity := tnet.RandIdentityOrFatal(t)
+
+	h := path.FromString("/dms3/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN")
+	err := publisher.Publish(context.Background(), identity.PrivateKey(), h)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	res, err := resolver.Resolve(context.Background(), identity.ID().Pretty())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if res != h {
+		t.Fatal("Got back incorrect value.")
+	}
+}
+
+func TestPrexistingExpiredRecord(t *testing.T) {
+	dstore := dssync.MutexWrap(ds.NewMapDatastore())
+	d := mockrouting.NewServer().ClientWithDatastore(context.Background(), tnet.RandIdentityOrFatal(t), dstore)
+
+	resolver := NewDms3NsResolver(d)
+	publisher := NewDms3NsPublisher(d, dstore)
+
+	identity := tnet.RandIdentityOrFatal(t)
+
+	// Make an expired record and put it in the datastore
+	h := path.FromString("/dms3/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN")
+	eol := time.Now().Add(time.Hour * -1)
+
+	entry, err := dms3ns.Create(identity.PrivateKey(), []byte(h), 0, eol)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = PutRecordToRouting(context.Background(), d, identity.PublicKey(), entry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Now, with an old record in the system already, try and publish a new one
+	err = publisher.Publish(context.Background(), identity.PrivateKey(), h)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = verifyCanResolve(resolver, identity.ID().Pretty(), h)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestPrexistingRecord(t *testing.T) {
+	dstore := dssync.MutexWrap(ds.NewMapDatastore())
+	d := mockrouting.NewServer().ClientWithDatastore(context.Background(), tnet.RandIdentityOrFatal(t), dstore)
+
+	resolver := NewDms3NsResolver(d)
+	publisher := NewDms3NsPublisher(d, dstore)
+
+	identity := tnet.RandIdentityOrFatal(t)
+
+	// Make a good record and put it in the datastore
+	h := path.FromString("/dms3/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN")
+	eol := time.Now().Add(time.Hour)
+	entry, err := dms3ns.Create(identity.PrivateKey(), []byte(h), 0, eol)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = PutRecordToRouting(context.Background(), d, identity.PublicKey(), entry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Now, with an old record in the system already, try and publish a new one
+	err = publisher.Publish(context.Background(), identity.PrivateKey(), h)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = verifyCanResolve(resolver, identity.ID().Pretty(), h)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func verifyCanResolve(r Resolver, name string, exp path.Path) error {
+	res, err := r.Resolve(context.Background(), name)
+	if err != nil {
+		return err
+	}
+
+	if res != exp {
+		return errors.New("got back wrong record")
+	}
+
+	return nil
+}

routing.go

+package namesys
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	proto "github.com/gogo/protobuf/proto"
+	mh "github.com/multiformats/go-multihash"
+	cid "gitlab.dms3.io/dms3/go-cid"
+	dms3ns "gitlab.dms3.io/dms3/go-dms3ns"
+	pb "gitlab.dms3.io/dms3/go-dms3ns/pb"
+	logging "gitlab.dms3.io/dms3/go-log"
+	path "gitlab.dms3.io/dms3/go-path"
+	opts "gitlab.dms3.io/dms3/interface-go-dms3-core/options/namesys"
+	peer "gitlab.dms3.io/p2p/go-p2p-core/peer"
+	routing "gitlab.dms3.io/p2p/go-p2p-core/routing"
+	dht "gitlab.dms3.io/p2p/go-p2p-kad-dht"
+)
+
+var log = logging.Logger("namesys")
+
+// Dms3NsResolver implements NSResolver for the main DMS3 SFS-like naming
+type Dms3NsResolver struct {
+	routing routing.ValueStore
+}
+
+// NewDms3NsResolver constructs a name resolver using the DMS3 Routing system
+// to implement SFS-like naming on top.
+func NewDms3NsResolver(route routing.ValueStore) *Dms3NsResolver {
+	if route == nil {
+		panic("attempt to create resolver with nil routing system")
+	}
+	return &Dms3NsResolver{
+		routing: route,
+	}
+}
+
+// Resolve implements Resolver.
+func (r *Dms3NsResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) {
+	return resolve(ctx, r, name, opts.ProcessOpts(options))
+}
+
+// ResolveAsync implements Resolver.
+func (r *Dms3NsResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result {
+	return resolveAsync(ctx, r, name, opts.ProcessOpts(options))
+}
+
+// resolveOnce implements resolver. Uses the DMS3 routing system to
+// resolve SFS-like names.
+func (r *Dms3NsResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
+	out := make(chan onceResult, 1)
+	log.Debugf("RoutingResolver resolving %s", name)
+	cancel := func() {}
+
+	if options.DhtTimeout != 0 {
+		// Resolution must complete within the timeout
+		ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout)
+	}
+
+	name = strings.TrimPrefix(name, "/dms3ns/")
+
+	pid, err := peer.Decode(name)
+	if err != nil {
+		log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err)
+		out <- onceResult{err: err}
+		close(out)
+		cancel()
+		return out
+	}
+
+	// Use the routing system to get the name.
+	// Note that the DHT will call the dms3ns validator when retrieving
+	// the value, which in turn verifies the dms3ns record signature
+	dms3nsKey := dms3ns.RecordKey(pid)
+
+	vals, err := r.routing.SearchValue(ctx, dms3nsKey, dht.Quorum(int(options.DhtRecordCount)))
+	if err != nil {
+		log.Debugf("RoutingResolver: dht get for name %s failed: %s", name, err)
+		out <- onceResult{err: err}
+		close(out)
+		cancel()
+		return out
+	}
+
+	go func() {
+		defer cancel()
+		defer close(out)
+		for {
+			select {
+			case val, ok := <-vals:
+				if !ok {
+					return
+				}
+
+				entry := new(pb.Dms3NsEntry)
+				err = proto.Unmarshal(val, entry)
+				if err != nil {
+					log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", name, err)
+					emitOnceResult(ctx, out, onceResult{err: err})
+					return
+				}
+
+				var p path.Path
+				// check for old style record:
+				if valh, err := mh.Cast(entry.GetValue()); err == nil {
+					// Its an old style multihash record
+					log.Debugf("encountered CIDv0 dms3ns entry: %s", valh)
+					p = path.FromCid(cid.NewCidV0(valh))
+				} else {
+					// Not a multihash, probably a new style record
+					p, err = path.ParsePath(string(entry.GetValue()))
+					if err != nil {
+						emitOnceResult(ctx, out, onceResult{err: err})
+						return
+					}
+				}
+
+				ttl := DefaultResolverCacheTTL
+				if entry.Ttl != nil {
+					ttl = time.Duration(*entry.Ttl)
+				}
+				switch eol, err := dms3ns.GetEOL(entry); err {
+				case dms3ns.ErrUnrecognizedValidity:
+					// No EOL.
+				case nil:
+					ttEol := time.Until(eol)
+					if ttEol < 0 {
+						// It *was* valid when we first resolved it.
+						ttl = 0
+					} else if ttEol < ttl {
+						ttl = ttEol
+					}
+				default:
+					log.Errorf("encountered error when parsing EOL: %s", err)
+					emitOnceResult(ctx, out, onceResult{err: err})
+					return
+				}
+
+				emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl})
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return out
+}