Merge branch 'port-2021-05-02' into 'master'

Port 2021 05 02

See merge request dms3/public/go-cidutil!1

.github/ISSUE_TEMPLATE/config.yml

+blank_issues_enabled: false
+contact_links:
+ - name: Getting Help on IPFS
+   url: https://ipfs.io/help
+   about: All information about how and where to get help on IPFS.
+ - name: IPFS Official Forum
+   url: https://discuss.ipfs.io
+   about: Please post general questions, support requests, and discussions here.

.github/ISSUE_TEMPLATE/open_an_issue.md

+---
+name: Open an issue
+about: Only for actionable issues relevant to this repository.
+title: ''
+labels: need/triage
+assignees: ''
+
+---
+<!--
+Hello! To ensure this issue is correctly addressed as soon as possible by the IPFS team, please try to make sure:
+
+- This issue is relevant to this repository's topic or codebase.
+
+- A clear description is provided. It should includes as much relevant information as possible and clear scope for the issue to be actionable.
+
+FOR GENERAL DISCUSSION, HELP OR QUESTIONS, please see the options at https://ipfs.io/help or head directly to https://discuss.ipfs.io.
+
+(you can delete this section after reading)
+-->

.github/config.yml

+# Configuration for welcome - https://github.com/behaviorbot/welcome
+
+# Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome
+# Comment to be posted to on first time issues
+newIssueWelcomeComment: >
+  Thank you for submitting your first issue to this repository! A maintainer
+  will be here shortly to triage and review.
+
+  In the meantime, please double-check that you have provided all the
+  necessary information to make this process easy! Any information that can
+  help save additional round trips is useful! We currently aim to give
+  initial feedback within **two business days**. If this does not happen, feel
+  free to leave a comment.
+
+  Please keep an eye on how this issue will be labeled, as labels give an
+  overview of priorities, assignments and additional actions requested by the
+  maintainers:
+
+    - "Priority" labels will show how urgent this is for the team.
+    - "Status" labels will show if this is ready to be worked on, blocked, or in progress.
+    - "Need" labels will indicate if additional input or analysis is required.
+
+  Finally, remember to use https://discuss.ipfs.io if you just need general
+  support.
+
+# Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome
+# Comment to be posted to on PRs from first time contributors in your repository
+newPRWelcomeComment: >
+  Thank you for submitting this PR!
+
+  A maintainer will be here shortly to review it.
+
+  We are super grateful, but we are also overloaded! Help us by making sure
+  that:
+
+    * The context for this PR is clear, with relevant discussion, decisions
+      and stakeholders linked/mentioned.
+
+    * Your contribution itself is clear (code comments, self-review for the
+      rest) and in its best form. Follow the [code contribution
+      guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md#code-contribution-guidelines)
+      if they apply.
+
+  Getting other community members to do a review would be great help too on
+  complex PRs (you can ask in the chats/forums). If you are unsure about
+  something, just leave us a comment.
+
+  Next steps:
+
+    * A maintainer will triage and assign priority to this PR, commenting on
+      any missing things and potentially assigning a reviewer for high
+      priority items.
+
+    * The PR gets reviews, discussed and approvals as needed.
+
+    * The PR is merged by maintainers when it has been approved and comments addressed.
+
+  We currently aim to provide initial feedback/triaging within **two business
+  days**. Please keep an eye on any labelling actions, as these will indicate
+  priorities and status of your contribution.
+
+  We are very grateful for your contribution!
+
+
+# Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge
+# Comment to be posted to on pull requests merged by a first time user
+# Currently disabled
+#firstPRMergeComment: ""

.gitlab-ci.yml

+stages:
+  - build
+  - test
+
+variables:
+  BUILD_DIR: "/tmp/$CI_CONCURRENT_PROJECT_ID"
+
+before_script:
+  - mkdir -p $BUILD_DIR/src
+  - cd $BUILD_DIR/src
+  - if [ -d $CI_PROJECT_DIR ]
+  - then
+  -    echo "soft link $CI_PROJECT_DIR exists"
+  - else
+  -    echo "creating soft link $CI_PROJECT_DIR"
+  -    ln -s $CI_PROJECT_DIR
+  - fi
+  - cd $CI_PROJECT_DIR
+
+build:
+  stage: build
+  tags:
+    - testing
+  script:
+    - echo $CI_JOB_STAGE
+    - go build
+
+test:
+  stage: test
+  tags:
+    - testing
+  script:
+    - echo $CI_JOB_STAGE
+    - go test -cover
+  coverage: '/coverage: \d+.\d+% of statements/'

.gx/lastpubver

+0.2.1: Qmf3gRH2L1QZy92gJHJEwKmBJKJGVf8RpN2kPPD2NQWg8G

.travis.yml

+os:
+  - linux
+
+language: go
+
+go:
+  - 1.11.x
+
+env:
+  global:
+    - GOTFLAGS="-race"
+  matrix:
+    - BUILD_DEPTYPE=gomod
+
+
+# disable travis install
+install:
+  - true
+
+script:
+  - bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh)
+
+
+cache:
+  directories:
+    - $GOPATH/pkg/mod
+    - $HOME/.cache/go-build
+
+notifications:
+  email: false

LICENSE

+The MIT License (MIT)
+
+Copyright (c) Protocol Labs, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

 # go-cidutil
 
+[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
+[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
+[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme)
+[![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-cidutil?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-cidutil)
+[![Build Status](https://travis-ci.org/ipfs/go-ipfs-cidutil.svg?branch=master)](https://travis-ci.org/ipfs/go-ipfs-cidutil)
+
+> go-cidutil implements various utilities and helper functions for working with CIDs
+
+## Lead Maintainer
+
+[Steven Allen](https://github.com/Stebalien)
+
+## Contribute
+
+PRs accepted.
+
+Small note: If editing the README, please conform to the [standard-readme](https://github.com/RichardLitt/standard-readme) specification.
+
+## License
+
+MIT © Protocol Labs, Inc.

ci/Jenkinsfile

+golang()

cid-fmt/main.go

+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	cidutil "gitlab.dms3.io/dms3/public/go-cidutil"
+
+	mb "github.com/multiformats/go-multibase"
+	c "gitlab.dms3.io/dms3/public/go-cid"
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "usage: %s [-b multibase-code] [-v cid-version] [--filter] <fmt-str> <cid> ...\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "--filter will read from stdin and convert anything that looks like a <cid>\n")
+	fmt.Fprintf(os.Stderr, "         -- including any non-cids that are valid Multihashes).\n")
+	fmt.Fprintf(os.Stderr, "<fmt-str> is either 'prefix' or a printf style format string:\n%s", cidutil.FormatRef)
+	os.Exit(2)
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+	}
+	newBase := mb.Encoding(-1)
+	var verConv func(cid c.Cid) (c.Cid, error)
+	args := os.Args[1:]
+	filter := false
+outer:
+	for len(args) > 0 {
+		switch args[0] {
+		case "-b":
+			if len(args) < 2 {
+				usage()
+			}
+			encoder, err := mb.EncoderByName(args[1])
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error())
+				os.Exit(2)
+			}
+			newBase = encoder.Encoding()
+			args = args[2:]
+		case "-v":
+			if len(args) < 2 {
+				usage()
+			}
+			switch args[1] {
+			case "0":
+				verConv = toCidV0
+			case "1":
+				verConv = toCidV1
+			default:
+				fmt.Fprintf(os.Stderr, "Error: Invalid cid version: %s\n", args[1])
+				os.Exit(2)
+			}
+			args = args[2:]
+		case "--filter":
+			filter = true
+			args = args[1:]
+		default:
+			break outer
+		}
+	}
+	if len(args) < 1 {
+		usage()
+	}
+	fmtStr := args[0]
+	switch fmtStr {
+	case "prefix":
+		fmtStr = "%P"
+	default:
+		if strings.IndexByte(fmtStr, '%') == -1 {
+			fmt.Fprintf(os.Stderr, "Error: Invalid format string: %s\n", fmtStr)
+			os.Exit(2)
+		}
+	}
+	format := func(cid c.Cid, cidStr string) (string, error) {
+		base := newBase
+		if base == -1 {
+			base, _ = c.ExtractEncoding(cidStr)
+		}
+		var err error
+		if verConv != nil {
+			cid, err = verConv(cid)
+			if err != nil {
+				return "", err
+			}
+		}
+		return cidutil.Format(fmtStr, base, cid)
+	}
+	if filter {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			buf := scanner.Bytes()
+			for {
+				i, j, cid, cidStr := cidutil.ScanForCid(buf)
+				os.Stdout.Write(buf[0:i])
+				if i == len(buf) {
+					os.Stdout.Write([]byte{'\n'})
+					break
+				}
+				str, err := format(cid, cidStr)
+				switch err.(type) {
+				case cidutil.FormatStringError:
+					fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+					os.Exit(2)
+				default:
+					// just use the orignal sting on non-fatal error
+					str = cidStr
+				case nil:
+				}
+				io.WriteString(os.Stdout, str)
+				buf = buf[j:]
+			}
+		}
+		if err := scanner.Err(); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(2)
+		}
+	} else {
+		for _, cidStr := range args[1:] {
+			cid, err := c.Decode(cidStr)
+			if err != nil {
+				fmt.Fprintf(os.Stdout, "!INVALID_CID!\n")
+				errorMsg("%s: %v", cidStr, err)
+				// Don't abort on a bad cid
+				continue
+			}
+			str, err := format(cid, cidStr)
+			switch err.(type) {
+			case cidutil.FormatStringError:
+				fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+				os.Exit(2)
+			default:
+				fmt.Fprintf(os.Stdout, "!ERROR!\n")
+				errorMsg("%s: %v", cidStr, err)
+				// Don't abort on cid specific errors
+				continue
+			case nil:
+				// no error
+			}
+			fmt.Fprintf(os.Stdout, "%s\n", str)
+		}
+	}
+	os.Exit(exitCode)
+}
+
+var exitCode = 0
+
+func errorMsg(fmtStr string, a ...interface{}) {
+	fmt.Fprintf(os.Stderr, "Error: ")
+	fmt.Fprintf(os.Stderr, fmtStr, a...)
+	fmt.Fprintf(os.Stderr, "\n")
+	exitCode = 1
+}
+
+func toCidV0(cid c.Cid) (c.Cid, error) {
+	if cid.Type() != c.DagProtobuf {
+		return c.Cid{}, fmt.Errorf("can't convert non-protobuf nodes to cidv0")
+	}
+	return c.NewCidV0(cid.Hash()), nil
+}
+
+func toCidV1(cid c.Cid) (c.Cid, error) {
+	return c.NewCidV1(cid.Type(), cid.Hash()), nil
+}

cid-fmt/main_test.go

+package main
+
+import (
+	"fmt"
+	"testing"
+
+	c "gitlab.dms3.io/dms3/public/go-cid"
+)
+
+func TestCidConv(t *testing.T) {
+	cidv0 := "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"
+	cidv1 := "bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354"
+	cid, err := c.Decode(cidv0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cid, err = toCidV1(cid)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cid.String() != cidv1 {
+		t.Fatalf("conversion failure: %s != %s", cid, cidv1)
+	}
+	cid, err = toCidV0(cid)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cidStr := cid.String()
+	if cidStr != cidv0 {
+		t.Error(fmt.Sprintf("conversion failure, expected: %s; but got: %s", cidv0, cidStr))
+	}
+}
+
+func TestBadCidConv(t *testing.T) {
+	// this cid is a raw leaf and should not be able to convert to cidv0
+	cidv1 := "bafkreifit7vvfkf2cwwzvyycdczm5znbdbqx54ab6shbesvwgkwthdf77y"
+	cid, err := c.Decode(cidv1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cid, err = toCidV0(cid)
+	if err == nil {
+		t.Fatal("expected failure")
+	}
+}

cidenc/encoder.go

+package cidenc
+
+import (
+	cid "github.com/ipfs/go-cid"
+	mbase "github.com/multiformats/go-multibase"
+)
+
+// Encoder is a basic Encoder that will encode CIDs using a specified
+// base and optionally upgrade a CIDv0 to CIDv1
+type Encoder struct {
+	Base    mbase.Encoder // The multibase to use
+	Upgrade bool          // If true upgrade CIDv0 to CIDv1 when encoding
+}
+
+// Default return a new default encoder
+func Default() Encoder {
+	return Encoder{Base: mbase.MustNewEncoder(mbase.Base32)}
+}
+
+// Encode encodes the cid using the parameters of the Encoder
+func (enc Encoder) Encode(c cid.Cid) string {
+	if enc.Upgrade && c.Version() == 0 {
+		c = cid.NewCidV1(c.Type(), c.Hash())
+	}
+	return c.Encode(enc.Base)
+}
+
+// Recode reencodes the cid string to match the parameters of the
+// encoder
+func (enc Encoder) Recode(v string) (string, error) {
+	skip, err := enc.noopRecode(v)
+	if skip || err != nil {
+		return v, err
+	}
+
+	c, err := cid.Decode(v)
+	if err != nil {
+		return v, err
+	}
+
+	return enc.Encode(c), nil
+}
+
+func (enc Encoder) noopRecode(v string) (bool, error) {
+	if len(v) < 2 {
+		return false, cid.ErrCidTooShort
+	}
+	ver := cidVer(v)
+	skip := ver == 0 && !enc.Upgrade || ver == 1 && v[0] == byte(enc.Base.Encoding())
+	return skip, nil
+}
+
+func cidVer(v string) int {
+	if len(v) == 46 && v[:2] == "Qm" {
+		return 0
+	} else {
+		return 1
+	}
+}

cidenc/encoder_test.go

+package cidenc
+
+import (
+	"testing"
+
+	cid "github.com/ipfs/go-cid"
+	mbase "github.com/multiformats/go-multibase"
+)
+
+func TestCidEncoder(t *testing.T) {
+	cidv0str := "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n"
+	cidv1str := "zdj7Wkkhxcu2rsiN6GUyHCLsSLL47kdUNfjbFqBUUhMFTZKBi"
+	cidb32str := "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"
+	cidv0, _ := cid.Decode(cidv0str)
+	cidv1, _ := cid.Decode(cidv1str)
+
+	testEncode := func(enc Encoder, cid cid.Cid, expect string) {
+		actual := enc.Encode(cid)
+		if actual != expect {
+			t.Errorf("%+v.Encode(%s): expected %s but got %s", enc, cid, expect, actual)
+		}
+	}
+
+	testRecode := func(enc Encoder, cid string, expect string) {
+		actual, err := enc.Recode(cid)
+		if err != nil {
+			t.Errorf("%+v.Recode(%s): %s", enc, cid, err)
+			return
+		}
+		if actual != expect {
+			t.Errorf("%+v.Recode(%s): expected %s but got %s", enc, cid, expect, actual)
+		}
+	}
+
+	enc := Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: false}
+	testEncode(enc, cidv0, cidv0str)
+	testEncode(enc, cidv1, cidv1str)
+	testRecode(enc, cidv0str, cidv0str)
+	testRecode(enc, cidv1str, cidv1str)
+	testRecode(enc, cidb32str, cidv1str)
+
+	enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: true}
+	testEncode(enc, cidv0, cidv1str)
+	testEncode(enc, cidv1, cidv1str)
+	testRecode(enc, cidv0str, cidv1str)
+	testRecode(enc, cidv1str, cidv1str)
+	testRecode(enc, cidb32str, cidv1str)
+
+	enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: false}
+	testEncode(enc, cidv0, cidv0str)
+	testEncode(enc, cidv1, cidb32str)
+	testRecode(enc, cidv0str, cidv0str)
+	testRecode(enc, cidv1str, cidb32str)
+	testRecode(enc, cidb32str, cidb32str)
+
+	enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: true}
+	testEncode(enc, cidv0, cidb32str)
+	testEncode(enc, cidv1, cidb32str)
+	testRecode(enc, cidv0str, cidb32str)
+	testRecode(enc, cidv1str, cidb32str)
+	testRecode(enc, cidb32str, cidb32str)
+
+	enc = Default()
+	testEncode(enc, cidv0, cidv0str)
+	testEncode(enc, cidv1, cidb32str)
+	testRecode(enc, cidv0str, cidv0str)
+	testRecode(enc, cidv1str, cidb32str)
+	testRecode(enc, cidb32str, cidb32str)
+
+}

codecov.yml

+coverage:
+  range: "50...100"
+comment: off

format.go

+package cidutil
+
+import (
+	"bytes"
+	"fmt"
+
+	mb "github.com/multiformats/go-multibase"
+	mh "github.com/multiformats/go-multihash"
+	c "gitlab.dms3.io/dms3/public/go-cid"
+)
+
+// FormatRef is a string documenting the format string for the Format function
+const FormatRef = `
+   %% literal %
+   %b multibase name
+   %B multibase code
+   %v version string
+   %V version number
+   %c codec name
+   %C codec code
+   %h multihash name
+   %H multihash code
+   %L hash digest length
+   %m multihash encoded in base %b (with multibase prefix)
+   %M multihash encoded in base %b without multibase prefix
+   %d hash digest encoded in base %b (with multibase prefix)
+   %D hash digest encoded in base %b without multibase prefix
+   %s cid string encoded in base %b (1)
+   %S cid string encoded in base %b without multibase prefix
+   %P cid prefix: %v-%c-%h-%L
+
+(1) For CID version 0 the multibase must be base58btc and no prefix is
+used.  For Cid version 1 the multibase prefix is included.
+`
+
+// Format formats a cid according to the format specificer as
+// documented in the FormatRef constant
+func Format(fmtStr string, base mb.Encoding, cid c.Cid) (string, error) {
+	p := cid.Prefix()
+	var out bytes.Buffer
+	var err error
+	encoder, err := mb.NewEncoder(base)
+	if err != nil {
+		return "", err
+	}
+	for i := 0; i < len(fmtStr); i++ {
+		if fmtStr[i] != '%' {
+			out.WriteByte(fmtStr[i])
+			continue
+		}
+		i++
+		if i >= len(fmtStr) {
+			return "", FormatStringError{"premature end of format string", ""}
+		}
+		switch fmtStr[i] {
+		case '%':
+			out.WriteByte('%')
+		case 'b': // base name
+			out.WriteString(baseToString(base))
+		case 'B': // base code
+			out.WriteByte(byte(base))
+		case 'v': // version string
+			fmt.Fprintf(&out, "cidv%d", p.Version)
+		case 'V': // version num
+			fmt.Fprintf(&out, "%d", p.Version)
+		case 'c': // codec name
+			out.WriteString(codecToString(p.Codec))
+		case 'C': // codec code
+			fmt.Fprintf(&out, "%d", p.Codec)
+		case 'h': // hash fun name
+			out.WriteString(hashToString(p.MhType))
+		case 'H': // hash fun code
+			fmt.Fprintf(&out, "%d", p.MhType)
+		case 'L': // hash length
+			fmt.Fprintf(&out, "%d", p.MhLength)
+		case 'm', 'M': // multihash encoded in base %b
+			out.WriteString(encode(encoder, cid.Hash(), fmtStr[i] == 'M'))
+		case 'd', 'D': // hash digest encoded in base %b
+			dec, err := mh.Decode(cid.Hash())
+			if err != nil {
+				return "", err
+			}
+			out.WriteString(encode(encoder, dec.Digest, fmtStr[i] == 'D'))
+		case 's': // cid string encoded in base %b
+			str, err := cid.StringOfBase(base)
+			if err != nil {
+				return "", err
+			}
+			out.WriteString(str)
+		case 'S': // cid string without base prefix
+			out.WriteString(encode(encoder, cid.Bytes(), true))
+		case 'P': // prefix
+			fmt.Fprintf(&out, "cidv%d-%s-%s-%d",
+				p.Version,
+				codecToString(p.Codec),
+				hashToString(p.MhType),
+				p.MhLength,
+			)
+		default:
+			return "", FormatStringError{"unrecognized specifier in format string", fmtStr[i-1 : i+1]}
+		}
+
+	}
+	return out.String(), err
+}
+
+// FormatStringError is the error return from Format when the format
+// string is ill formed
+type FormatStringError struct {
+	Message   string
+	Specifier string
+}
+
+func (e FormatStringError) Error() string {
+	if e.Specifier == "" {
+		return e.Message
+	} else {
+		return fmt.Sprintf("%s: %s", e.Message, e.Specifier)
+	}
+}
+
+func baseToString(base mb.Encoding) string {
+	baseStr, ok := mb.EncodingToStr[base]
+	if !ok {
+		return fmt.Sprintf("base?%c", base)
+	}
+	return baseStr
+}
+
+func codecToString(num uint64) string {
+	name, ok := c.CodecToStr[num]
+	if !ok {
+		return fmt.Sprintf("codec?%d", num)
+	}
+	return name
+}
+
+func hashToString(num uint64) string {
+	name, ok := mh.Codes[num]
+	if !ok {
+		return fmt.Sprintf("hash?%d", num)
+	}
+	return name
+}
+
+func encode(base mb.Encoder, data []byte, strip bool) string {
+	str := base.Encode(data)
+	if strip {
+		return str[1:]
+	}
+	return str
+}
+
+// ScanForCid scans bytes for anything resembling a CID. If one is
+// found `i` will point to the begging of the cid and `j` to to the
+// end and the cid will be returned, otherwise `i` and `j` will point
+// the end of the buffer and the cid will be `Undef`.
+func ScanForCid(buf []byte) (i, j int, cid c.Cid, cidStr string) {
+	i = 0
+	for {
+		i = j
+		for i < len(buf) && !asciiIsAlpha(buf[i]) {
+			i++
+		}
+		j = i
+		if i == len(buf) {
+			return
+		}
+		for j < len(buf) && asciiIsAlpha(buf[j]) {
+			j++
+		}
+		if j-i <= 1 || j-i > 128 || !supported[buf[i]] {
+			continue
+		}
+		var err error
+		cidStr = string(buf[i:j])
+		cid, err = c.Decode(cidStr)
+		if err == nil {
+			return
+		}
+	}
+}
+
+var supported = make([]bool, 256)
+
+func init() {
+	// for now base64 encoding are not supported as they contain non
+	// alhphanumeric characters
+	supportedPrefixes := []byte("QfFbBcCvVtThzZ")
+	for _, b := range supportedPrefixes {
+		supported[b] = true
+	}
+}
+
+func asciiIsAlpha(b byte) bool {
+	return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') || ('0' <= b && b <= '9')
+}

format_test.go

+package cidutil
+
+import (
+	"fmt"
+	"testing"
+
+	mb "github.com/multiformats/go-multibase"
+	c "gitlab.dms3.io/dms3/public/go-cid"
+)
+
+func TestFmt(t *testing.T) {
+	cids := map[string]string{
+		"cidv0": "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn",
+		"cidv1": "zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD",
+	}
+	tests := []struct {
+		cidId   string
+		newBase mb.Encoding
+		fmtStr  string
+		result  string
+	}{
+		{"cidv0", -1, "%P", "cidv0-protobuf-sha2-256-32"},
+		{"cidv0", -1, "%b-%v-%c-%h-%L", "base58btc-cidv0-protobuf-sha2-256-32"},
+		{"cidv0", -1, "%s", "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"},
+		{"cidv0", -1, "%S", "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"},
+		{"cidv0", -1, "ver#%V/#%C/#%H/%L", "ver#0/#112/#18/32"},
+		{"cidv0", -1, "%m", "zQmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"},
+		{"cidv0", -1, "%M", "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"},
+		{"cidv0", -1, "%d", "z72gdmFAgRzYHkJzKiL8MgMMRW3BTSCGyDHroPxJbxMJn"},
+		{"cidv0", -1, "%D", "72gdmFAgRzYHkJzKiL8MgMMRW3BTSCGyDHroPxJbxMJn"},
+		{"cidv0", 'B', "%S", "CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y"},
+		{"cidv0", 'B', "%B%S", "BCIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y"},
+		{"cidv1", -1, "%P", "cidv1-protobuf-sha2-256-32"},
+		{"cidv1", -1, "%b-%v-%c-%h-%L", "base58btc-cidv1-protobuf-sha2-256-32"},
+		{"cidv1", -1, "%s", "zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD"},
+		{"cidv1", -1, "%S", "dj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD"},
+		{"cidv1", -1, "ver#%V/#%C/#%H/%L", "ver#1/#112/#18/32"},
+		{"cidv1", -1, "%m", "zQmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H"},
+		{"cidv1", -1, "%M", "QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H"},
+		{"cidv1", -1, "%d", "zAux4gVVsLRMXtsZ9fd3tFEZN4jGYB6kP37fgoZNTc11H"},
+		{"cidv1", -1, "%D", "Aux4gVVsLRMXtsZ9fd3tFEZN4jGYB6kP37fgoZNTc11H"},
+		{"cidv1", 'B', "%s", "BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA"},
+		{"cidv1", 'B', "%S", "AFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA"},
+		{"cidv1", 'B', "%B%S", "BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA"},
+	}
+	for _, tc := range tests {
+		name := fmt.Sprintf("%s/%s", tc.cidId, tc.fmtStr)
+		if tc.newBase != -1 {
+			name = fmt.Sprintf("%s/%c", name, tc.newBase)
+		}
+		cidStr := cids[tc.cidId]
+		t.Run(name, func(t *testing.T) {
+			testFmt(t, cidStr, tc.newBase, tc.fmtStr, tc.result)
+		})
+	}
+}
+
+func testFmt(t *testing.T, cidStr string, newBase mb.Encoding, fmtStr string, result string) {
+	cid, err := c.Decode(cidStr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	base := newBase
+	if newBase == -1 {
+		base, _ = c.ExtractEncoding(cidStr)
+	}
+	str, err := Format(fmtStr, base, cid)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if str != result {
+		t.Error(fmt.Sprintf("expected: %s; but got: %s", result, str))
+	}
+}
+
+func TestScanForCid(t *testing.T) {
+	testStr := []byte(`
+/ipfs/QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H 22 45
+/ipfs/zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD/foobar
+BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA
+skip me (too long): QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H876
+skip me (too short): bafybeietjgsrl3eqpqpcabv3g6iubytsifvq
+bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja
+bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja.
+`)
+	cids := []string{
+		"QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H",
+		"zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD",
+		"BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA",
+		"bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja",
+		"bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja",
+	}
+
+	buf := testStr
+	idx := 0
+	offset := 0
+	for len(buf) > 0 {
+		_, j, _, cidStr := ScanForCid(buf)
+		if cidStr != "" && cids[idx] != cidStr {
+			t.Fatalf("Scan failed, expected %s, got %s (idx=%d offset=%d)", cids[idx], cidStr, idx, offset)
+		}
+		buf = buf[j:]
+		offset += j
+		idx++
+	}
+}

go.mod

+module gitlab.dms3.io/dms3/public/go-cidutil
+
+require (
+	github.com/multiformats/go-multibase v0.0.3
+	github.com/multiformats/go-multihash v0.0.14
+	gitlab.dms3.io/dms3/public/go-cid v0.0.1
+)
+
+go 1.15

go.sum

+github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyFSs7UnsU=
+github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
+github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc=
+github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
+github.com/ipfs/go-cid v0.0.2 h1:tuuKaZPU1M6HcejsO3AcYWW8sZ8MTvyxfc4uqB4eFE8=
+github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms=
+github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
+github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
+github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
+github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU=
+github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
+github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
+github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16 h1:5W7KhL8HVF3XCFOweFD3BNESdnO8ewyYTFT2R+/b8FQ=
+github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ=
+github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
+github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78=
+github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
+github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
+github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
+github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA=
+github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
+github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
+github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
+github.com/multiformats/go-multihash v0.0.1 h1:HHwN1K12I+XllBCrqKnhX949Orn4oawPkegHMu2vDqQ=
+github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
+github.com/multiformats/go-multihash v0.0.8 h1:wrYcW5yxSi3dU07n5jnuS5PrNwyHy0zRHGVoUugWvXg=
+github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
+github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4=
+github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
+github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM=
+github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
+github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
+github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
+github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
+github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
+github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+gitlab.dms3.io/dms3/public/go-cid v0.0.0 h1:DKvGTYv5GqSEeVLfQAHBbsFZuWCZFNK5BHdBFykF45E=
+gitlab.dms3.io/dms3/public/go-cid v0.0.0/go.mod h1:3WDixn/g/St71Wop8EY5QsPF3IHBOxb4VIgwWeOl5BQ=
+gitlab.dms3.io/dms3/public/go-cid v0.0.1 h1:qs4dtkDigcLGY/58dIZaFjKLt+orrTcmTBvtqaM3570=
+gitlab.dms3.io/dms3/public/go-cid v0.0.1/go.mod h1:GQw3gc4CSrFY+aX6M+OBQDlg0p5/eQJoRrayaZzkAOQ=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE=
+golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190219092855-153ac476189d h1:Z0Ahzd7HltpJtjAHHxX8QFP3j1yYgiuvjbjRzDj/KH0=
+golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

inline.go

+package cidutil
+
+import (
+	mhash "github.com/multiformats/go-multihash"
+	cid "gitlab.dms3.io/dms3/public/go-cid"
+)
+
+// InlineBuilder is a cid.Builder that will use the id multihash when the
+// size of the content is no more than limit
+type InlineBuilder struct {
+	cid.Builder     // Parent Builder
+	Limit       int // Limit (inclusive)
+}
+
+// WithCodec implements the cid.Builder interface
+func (p InlineBuilder) WithCodec(c uint64) cid.Builder {
+	return InlineBuilder{p.Builder.WithCodec(c), p.Limit}
+}
+
+// Sum implements the cid.Builder interface
+func (p InlineBuilder) Sum(data []byte) (cid.Cid, error) {
+	if len(data) > p.Limit {
+		return p.Builder.Sum(data)
+	}
+	return cid.V1Builder{Codec: p.GetCodec(), MhType: mhash.ID}.Sum(data)
+}

inline_test.go

+package cidutil
+
+import (
+	"math/rand"
+	"testing"
+
+	mhash "github.com/multiformats/go-multihash"
+	cid "gitlab.dms3.io/dms3/public/go-cid"
+)
+
+func TestInlineBuilderSmallValue(t *testing.T) {
+	builder := InlineBuilder{cid.V0Builder{}, 64}
+	c, err := builder.Sum([]byte("Hello World"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.Prefix().MhType != mhash.ID {
+		t.Fatal("Inliner builder failed to use ID Multihash on small values")
+	}
+}
+
+func TestInlinerBuilderLargeValue(t *testing.T) {
+	builder := InlineBuilder{cid.V0Builder{}, 64}
+	data := make([]byte, 512)
+	rand.Read(data)
+	c, err := builder.Sum(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.Prefix().MhType == mhash.ID {
+		t.Fatal("Inliner builder used ID Multihash on large values")
+	}
+}

set.go

+package cidutil
+
+import (
+	"context"
+
+	c "gitlab.dms3.io/dms3/public/go-cid"
+)
+
+type Set = c.Set
+
+func NewSet() *Set { return c.NewSet() }
+
+// StreamingSet is an extension of Set which allows to implement back-pressure
+// for the Visit function
+type StreamingSet struct {
+	Set *Set
+	New chan c.Cid
+}
+
+// NewStreamingSet initializes and returns new Set.
+func NewStreamingSet() *StreamingSet {
+	return &StreamingSet{
+		Set: c.NewSet(),
+		New: make(chan c.Cid),
+	}
+}
+
+// Visitor creates new visitor which adds a Cids to the set and emits them to
+// the set.New channel
+func (s *StreamingSet) Visitor(ctx context.Context) func(c c.Cid) bool {
+	return func(c c.Cid) bool {
+		if s.Set.Visit(c) {
+			select {
+			case s.New <- c:
+			case <-ctx.Done():
+			}
+			return true
+		}
+
+		return false
+	}
+}

slice.go

+package cidutil
+
+import (
+	"sort"
+
+	"gitlab.dms3.io/dms3/public/go-cid"
+)
+
+// Slice is a convenience type for sorting CIDs
+type Slice []cid.Cid
+
+func (s Slice) Len() int {
+	return len(s)
+}
+
+func (s Slice) Less(i, j int) bool {
+	return s[i].KeyString() < s[j].KeyString()
+}
+
+func (s Slice) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s Slice) Sort() {
+	sort.Sort(s)
+}
+
+// Sort sorts a slice of CIDs
+func Sort(s []cid.Cid) {
+	Slice(s).Sort()
+}