diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..4b86d7197f9a6f1a988c503defc8451392cd5e55 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Getting Help on IPFS + url: https://ipfs.io/help + about: All information about how and where to get help on IPFS. + - name: IPFS Official Forum + url: https://discuss.ipfs.io + about: Please post general questions, support requests, and discussions here. diff --git a/.github/ISSUE_TEMPLATE/open_an_issue.md b/.github/ISSUE_TEMPLATE/open_an_issue.md new file mode 100644 index 0000000000000000000000000000000000000000..4fcbd00aca0d513c2729d8df4afb5a01fdbe7d02 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/open_an_issue.md @@ -0,0 +1,19 @@ +--- +name: Open an issue +about: Only for actionable issues relevant to this repository. +title: '' +labels: need/triage +assignees: '' + +--- + diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..ed26646a0f7cdda6cf10ede2b0b98cac89cf67b0 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,68 @@ +# Configuration for welcome - https://github.com/behaviorbot/welcome + +# Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome +# Comment to be posted to on first time issues +newIssueWelcomeComment: > + Thank you for submitting your first issue to this repository! A maintainer + will be here shortly to triage and review. + + In the meantime, please double-check that you have provided all the + necessary information to make this process easy! Any information that can + help save additional round trips is useful! We currently aim to give + initial feedback within **two business days**. If this does not happen, feel + free to leave a comment. + + Please keep an eye on how this issue will be labeled, as labels give an + overview of priorities, assignments and additional actions requested by the + maintainers: + + - "Priority" labels will show how urgent this is for the team. + - "Status" labels will show if this is ready to be worked on, blocked, or in progress. + - "Need" labels will indicate if additional input or analysis is required. + + Finally, remember to use https://discuss.ipfs.io if you just need general + support. + +# Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome +# Comment to be posted to on PRs from first time contributors in your repository +newPRWelcomeComment: > + Thank you for submitting this PR! + + A maintainer will be here shortly to review it. + + We are super grateful, but we are also overloaded! Help us by making sure + that: + + * The context for this PR is clear, with relevant discussion, decisions + and stakeholders linked/mentioned. + + * Your contribution itself is clear (code comments, self-review for the + rest) and in its best form. Follow the [code contribution + guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md#code-contribution-guidelines) + if they apply. + + Getting other community members to do a review would be great help too on + complex PRs (you can ask in the chats/forums). If you are unsure about + something, just leave us a comment. + + Next steps: + + * A maintainer will triage and assign priority to this PR, commenting on + any missing things and potentially assigning a reviewer for high + priority items. + + * The PR gets reviews, discussed and approvals as needed. + + * The PR is merged by maintainers when it has been approved and comments addressed. + + We currently aim to provide initial feedback/triaging within **two business + days**. Please keep an eye on any labelling actions, as these will indicate + priorities and status of your contribution. + + We are very grateful for your contribution! + + +# Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge +# Comment to be posted to on pull requests merged by a first time user +# Currently disabled +#firstPRMergeComment: "" diff --git a/.gx/lastpubver b/.gx/lastpubver new file mode 100644 index 0000000000000000000000000000000000000000..3c0f0af3f4758a489932c29f0aa5bceba94deb48 --- /dev/null +++ b/.gx/lastpubver @@ -0,0 +1 @@ +0.2.1: Qmf3gRH2L1QZy92gJHJEwKmBJKJGVf8RpN2kPPD2NQWg8G diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000000000000000000000000000000000..5163d693fc757ec0c13d134e9d4cabd6cc7c85d7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,30 @@ +os: + - linux + +language: go + +go: + - 1.11.x + +env: + global: + - GOTFLAGS="-race" + matrix: + - BUILD_DEPTYPE=gomod + + +# disable travis install +install: + - true + +script: + - bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh) + + +cache: + directories: + - $GOPATH/pkg/mod + - $HOME/.cache/go-build + +notifications: + email: false diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..083bc1d0d7749c2a40b33ba6c152eb8d381f31d6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Protocol Labs, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md index 4a98251cfe5c6205f932152f79bfc059a8bd5d96..d0c17f3aa80eee56d4b1676d2993a23e80f6b411 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,23 @@ # go-cidutil +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) +[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) +[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) +[![GoDoc](https://godoc.org/github.com/ipfs/go-ipfs-cidutil?status.svg)](https://godoc.org/github.com/ipfs/go-ipfs-cidutil) +[![Build Status](https://travis-ci.org/ipfs/go-ipfs-cidutil.svg?branch=master)](https://travis-ci.org/ipfs/go-ipfs-cidutil) + +> go-cidutil implements various utilities and helper functions for working with CIDs + +## Lead Maintainer + +[Steven Allen](https://github.com/Stebalien) + +## Contribute + +PRs accepted. + +Small note: If editing the README, please conform to the [standard-readme](https://github.com/RichardLitt/standard-readme) specification. + +## License + +MIT © Protocol Labs, Inc. diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile new file mode 100644 index 0000000000000000000000000000000000000000..b2067e6232a4e61367aa25960899afdff1eba73c --- /dev/null +++ b/ci/Jenkinsfile @@ -0,0 +1 @@ +golang() diff --git a/cid-fmt/main.go b/cid-fmt/main.go new file mode 100644 index 0000000000000000000000000000000000000000..873565823881597f902e70c400b3fb31914bdbae --- /dev/null +++ b/cid-fmt/main.go @@ -0,0 +1,169 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "strings" + + cidutil "github.com/ipfs/go-cidutil" + + c "github.com/ipfs/go-cid" + mb "github.com/multiformats/go-multibase" +) + +func usage() { + fmt.Fprintf(os.Stderr, "usage: %s [-b multibase-code] [-v cid-version] [--filter] ...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "--filter will read from stdin and convert anything that looks like a \n") + fmt.Fprintf(os.Stderr, " -- including any non-cids that are valid Multihashes).\n") + fmt.Fprintf(os.Stderr, " is either 'prefix' or a printf style format string:\n%s", cidutil.FormatRef) + os.Exit(2) +} + +func main() { + if len(os.Args) < 2 { + usage() + } + newBase := mb.Encoding(-1) + var verConv func(cid c.Cid) (c.Cid, error) + args := os.Args[1:] + filter := false +outer: + for len(args) > 0 { + switch args[0] { + case "-b": + if len(args) < 2 { + usage() + } + encoder, err := mb.EncoderByName(args[1]) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) + os.Exit(2) + } + newBase = encoder.Encoding() + args = args[2:] + case "-v": + if len(args) < 2 { + usage() + } + switch args[1] { + case "0": + verConv = toCidV0 + case "1": + verConv = toCidV1 + default: + fmt.Fprintf(os.Stderr, "Error: Invalid cid version: %s\n", args[1]) + os.Exit(2) + } + args = args[2:] + case "--filter": + filter = true + args = args[1:] + default: + break outer + } + } + if len(args) < 1 { + usage() + } + fmtStr := args[0] + switch fmtStr { + case "prefix": + fmtStr = "%P" + default: + if strings.IndexByte(fmtStr, '%') == -1 { + fmt.Fprintf(os.Stderr, "Error: Invalid format string: %s\n", fmtStr) + os.Exit(2) + } + } + format := func(cid c.Cid, cidStr string) (string, error) { + base := newBase + if base == -1 { + base, _ = c.ExtractEncoding(cidStr) + } + var err error + if verConv != nil { + cid, err = verConv(cid) + if err != nil { + return "", err + } + } + return cidutil.Format(fmtStr, base, cid) + } + if filter { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + buf := scanner.Bytes() + for { + i, j, cid, cidStr := cidutil.ScanForCid(buf) + os.Stdout.Write(buf[0:i]) + if i == len(buf) { + os.Stdout.Write([]byte{'\n'}) + break + } + str, err := format(cid, cidStr) + switch err.(type) { + case cidutil.FormatStringError: + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(2) + default: + // just use the orignal sting on non-fatal error + str = cidStr + case nil: + } + io.WriteString(os.Stdout, str) + buf = buf[j:] + } + } + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(2) + } + } else { + for _, cidStr := range args[1:] { + cid, err := c.Decode(cidStr) + if err != nil { + fmt.Fprintf(os.Stdout, "!INVALID_CID!\n") + errorMsg("%s: %v", cidStr, err) + // Don't abort on a bad cid + continue + } + str, err := format(cid, cidStr) + switch err.(type) { + case cidutil.FormatStringError: + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(2) + default: + fmt.Fprintf(os.Stdout, "!ERROR!\n") + errorMsg("%s: %v", cidStr, err) + // Don't abort on cid specific errors + continue + case nil: + // no error + } + fmt.Fprintf(os.Stdout, "%s\n", str) + } + } + os.Exit(exitCode) +} + +var exitCode = 0 + +func errorMsg(fmtStr string, a ...interface{}) { + fmt.Fprintf(os.Stderr, "Error: ") + fmt.Fprintf(os.Stderr, fmtStr, a...) + fmt.Fprintf(os.Stderr, "\n") + exitCode = 1 +} + +func toCidV0(cid c.Cid) (c.Cid, error) { + if cid.Type() != c.DagProtobuf { + return c.Cid{}, fmt.Errorf("can't convert non-protobuf nodes to cidv0") + } + return c.NewCidV0(cid.Hash()), nil +} + +func toCidV1(cid c.Cid) (c.Cid, error) { + return c.NewCidV1(cid.Type(), cid.Hash()), nil +} diff --git a/cid-fmt/main_test.go b/cid-fmt/main_test.go new file mode 100644 index 0000000000000000000000000000000000000000..5609cd2666fd976c29c08c00e83322497d95fdfe --- /dev/null +++ b/cid-fmt/main_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + "testing" + + c "github.com/ipfs/go-cid" +) + +func TestCidConv(t *testing.T) { + cidv0 := "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" + cidv1 := "bafybeiczsscdsbs7ffqz55asqdf3smv6klcw3gofszvwlyarci47bgf354" + cid, err := c.Decode(cidv0) + if err != nil { + t.Fatal(err) + } + cid, err = toCidV1(cid) + if err != nil { + t.Fatal(err) + } + if cid.String() != cidv1 { + t.Fatalf("conversion failure: %s != %s", cid, cidv1) + } + cid, err = toCidV0(cid) + if err != nil { + t.Fatal(err) + } + cidStr := cid.String() + if cidStr != cidv0 { + t.Error(fmt.Sprintf("conversion failure, expected: %s; but got: %s", cidv0, cidStr)) + } +} + +func TestBadCidConv(t *testing.T) { + // this cid is a raw leaf and should not be able to convert to cidv0 + cidv1 := "bafkreifit7vvfkf2cwwzvyycdczm5znbdbqx54ab6shbesvwgkwthdf77y" + cid, err := c.Decode(cidv1) + if err != nil { + t.Fatal(err) + } + cid, err = toCidV0(cid) + if err == nil { + t.Fatal("expected failure") + } +} diff --git a/cidenc/encoder.go b/cidenc/encoder.go new file mode 100644 index 0000000000000000000000000000000000000000..83ef0168756fa9a422f851e6c4f95531cfbf389d --- /dev/null +++ b/cidenc/encoder.go @@ -0,0 +1,59 @@ +package cidenc + +import ( + cid "github.com/ipfs/go-cid" + mbase "github.com/multiformats/go-multibase" +) + +// Encoder is a basic Encoder that will encode CIDs using a specified +// base and optionally upgrade a CIDv0 to CIDv1 +type Encoder struct { + Base mbase.Encoder // The multibase to use + Upgrade bool // If true upgrade CIDv0 to CIDv1 when encoding +} + +// Default return a new default encoder +func Default() Encoder { + return Encoder{Base: mbase.MustNewEncoder(mbase.Base32)} +} + +// Encode encodes the cid using the parameters of the Encoder +func (enc Encoder) Encode(c cid.Cid) string { + if enc.Upgrade && c.Version() == 0 { + c = cid.NewCidV1(c.Type(), c.Hash()) + } + return c.Encode(enc.Base) +} + +// Recode reencodes the cid string to match the parameters of the +// encoder +func (enc Encoder) Recode(v string) (string, error) { + skip, err := enc.noopRecode(v) + if skip || err != nil { + return v, err + } + + c, err := cid.Decode(v) + if err != nil { + return v, err + } + + return enc.Encode(c), nil +} + +func (enc Encoder) noopRecode(v string) (bool, error) { + if len(v) < 2 { + return false, cid.ErrCidTooShort + } + ver := cidVer(v) + skip := ver == 0 && !enc.Upgrade || ver == 1 && v[0] == byte(enc.Base.Encoding()) + return skip, nil +} + +func cidVer(v string) int { + if len(v) == 46 && v[:2] == "Qm" { + return 0 + } else { + return 1 + } +} diff --git a/cidenc/encoder_test.go b/cidenc/encoder_test.go new file mode 100644 index 0000000000000000000000000000000000000000..225aa1f0c166622d4e63a9bc9d69115ec4ddeab4 --- /dev/null +++ b/cidenc/encoder_test.go @@ -0,0 +1,70 @@ +package cidenc + +import ( + "testing" + + cid "github.com/ipfs/go-cid" + mbase "github.com/multiformats/go-multibase" +) + +func TestCidEncoder(t *testing.T) { + cidv0str := "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + cidv1str := "zdj7Wkkhxcu2rsiN6GUyHCLsSLL47kdUNfjbFqBUUhMFTZKBi" + cidb32str := "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + cidv0, _ := cid.Decode(cidv0str) + cidv1, _ := cid.Decode(cidv1str) + + testEncode := func(enc Encoder, cid cid.Cid, expect string) { + actual := enc.Encode(cid) + if actual != expect { + t.Errorf("%+v.Encode(%s): expected %s but got %s", enc, cid, expect, actual) + } + } + + testRecode := func(enc Encoder, cid string, expect string) { + actual, err := enc.Recode(cid) + if err != nil { + t.Errorf("%+v.Recode(%s): %s", enc, cid, err) + return + } + if actual != expect { + t.Errorf("%+v.Recode(%s): expected %s but got %s", enc, cid, expect, actual) + } + } + + enc := Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: false} + testEncode(enc, cidv0, cidv0str) + testEncode(enc, cidv1, cidv1str) + testRecode(enc, cidv0str, cidv0str) + testRecode(enc, cidv1str, cidv1str) + testRecode(enc, cidb32str, cidv1str) + + enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: true} + testEncode(enc, cidv0, cidv1str) + testEncode(enc, cidv1, cidv1str) + testRecode(enc, cidv0str, cidv1str) + testRecode(enc, cidv1str, cidv1str) + testRecode(enc, cidb32str, cidv1str) + + enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: false} + testEncode(enc, cidv0, cidv0str) + testEncode(enc, cidv1, cidb32str) + testRecode(enc, cidv0str, cidv0str) + testRecode(enc, cidv1str, cidb32str) + testRecode(enc, cidb32str, cidb32str) + + enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: true} + testEncode(enc, cidv0, cidb32str) + testEncode(enc, cidv1, cidb32str) + testRecode(enc, cidv0str, cidb32str) + testRecode(enc, cidv1str, cidb32str) + testRecode(enc, cidb32str, cidb32str) + + enc = Default() + testEncode(enc, cidv0, cidv0str) + testEncode(enc, cidv1, cidb32str) + testRecode(enc, cidv0str, cidv0str) + testRecode(enc, cidv1str, cidb32str) + testRecode(enc, cidb32str, cidb32str) + +} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000000000000000000000000000000000000..5f88a9ea2785f8dfafe65d5c5fa9663de93ff423 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,3 @@ +coverage: + range: "50...100" +comment: off diff --git a/format.go b/format.go new file mode 100644 index 0000000000000000000000000000000000000000..21e157df2a9341aed002c1750d14f0561a203468 --- /dev/null +++ b/format.go @@ -0,0 +1,197 @@ +package cidutil + +import ( + "bytes" + "fmt" + + c "github.com/ipfs/go-cid" + mb "github.com/multiformats/go-multibase" + mh "github.com/multiformats/go-multihash" +) + +// FormatRef is a string documenting the format string for the Format function +const FormatRef = ` + %% literal % + %b multibase name + %B multibase code + %v version string + %V version number + %c codec name + %C codec code + %h multihash name + %H multihash code + %L hash digest length + %m multihash encoded in base %b (with multibase prefix) + %M multihash encoded in base %b without multibase prefix + %d hash digest encoded in base %b (with multibase prefix) + %D hash digest encoded in base %b without multibase prefix + %s cid string encoded in base %b (1) + %S cid string encoded in base %b without multibase prefix + %P cid prefix: %v-%c-%h-%L + +(1) For CID version 0 the multibase must be base58btc and no prefix is +used. For Cid version 1 the multibase prefix is included. +` + +// Format formats a cid according to the format specificer as +// documented in the FormatRef constant +func Format(fmtStr string, base mb.Encoding, cid c.Cid) (string, error) { + p := cid.Prefix() + var out bytes.Buffer + var err error + encoder, err := mb.NewEncoder(base) + if err != nil { + return "", err + } + for i := 0; i < len(fmtStr); i++ { + if fmtStr[i] != '%' { + out.WriteByte(fmtStr[i]) + continue + } + i++ + if i >= len(fmtStr) { + return "", FormatStringError{"premature end of format string", ""} + } + switch fmtStr[i] { + case '%': + out.WriteByte('%') + case 'b': // base name + out.WriteString(baseToString(base)) + case 'B': // base code + out.WriteByte(byte(base)) + case 'v': // version string + fmt.Fprintf(&out, "cidv%d", p.Version) + case 'V': // version num + fmt.Fprintf(&out, "%d", p.Version) + case 'c': // codec name + out.WriteString(codecToString(p.Codec)) + case 'C': // codec code + fmt.Fprintf(&out, "%d", p.Codec) + case 'h': // hash fun name + out.WriteString(hashToString(p.MhType)) + case 'H': // hash fun code + fmt.Fprintf(&out, "%d", p.MhType) + case 'L': // hash length + fmt.Fprintf(&out, "%d", p.MhLength) + case 'm', 'M': // multihash encoded in base %b + out.WriteString(encode(encoder, cid.Hash(), fmtStr[i] == 'M')) + case 'd', 'D': // hash digest encoded in base %b + dec, err := mh.Decode(cid.Hash()) + if err != nil { + return "", err + } + out.WriteString(encode(encoder, dec.Digest, fmtStr[i] == 'D')) + case 's': // cid string encoded in base %b + str, err := cid.StringOfBase(base) + if err != nil { + return "", err + } + out.WriteString(str) + case 'S': // cid string without base prefix + out.WriteString(encode(encoder, cid.Bytes(), true)) + case 'P': // prefix + fmt.Fprintf(&out, "cidv%d-%s-%s-%d", + p.Version, + codecToString(p.Codec), + hashToString(p.MhType), + p.MhLength, + ) + default: + return "", FormatStringError{"unrecognized specifier in format string", fmtStr[i-1 : i+1]} + } + + } + return out.String(), err +} + +// FormatStringError is the error return from Format when the format +// string is ill formed +type FormatStringError struct { + Message string + Specifier string +} + +func (e FormatStringError) Error() string { + if e.Specifier == "" { + return e.Message + } else { + return fmt.Sprintf("%s: %s", e.Message, e.Specifier) + } +} + +func baseToString(base mb.Encoding) string { + baseStr, ok := mb.EncodingToStr[base] + if !ok { + return fmt.Sprintf("base?%c", base) + } + return baseStr +} + +func codecToString(num uint64) string { + name, ok := c.CodecToStr[num] + if !ok { + return fmt.Sprintf("codec?%d", num) + } + return name +} + +func hashToString(num uint64) string { + name, ok := mh.Codes[num] + if !ok { + return fmt.Sprintf("hash?%d", num) + } + return name +} + +func encode(base mb.Encoder, data []byte, strip bool) string { + str := base.Encode(data) + if strip { + return str[1:] + } + return str +} + +// ScanForCid scans bytes for anything resembling a CID. If one is +// found `i` will point to the begging of the cid and `j` to to the +// end and the cid will be returned, otherwise `i` and `j` will point +// the end of the buffer and the cid will be `Undef`. +func ScanForCid(buf []byte) (i, j int, cid c.Cid, cidStr string) { + i = 0 + for { + i = j + for i < len(buf) && !asciiIsAlpha(buf[i]) { + i++ + } + j = i + if i == len(buf) { + return + } + for j < len(buf) && asciiIsAlpha(buf[j]) { + j++ + } + if j-i <= 1 || j-i > 128 || !supported[buf[i]] { + continue + } + var err error + cidStr = string(buf[i:j]) + cid, err = c.Decode(cidStr) + if err == nil { + return + } + } +} + +var supported = make([]bool, 256) + +func init() { + // for now base64 encoding are not supported as they contain non + // alhphanumeric characters + supportedPrefixes := []byte("QfFbBcCvVtThzZ") + for _, b := range supportedPrefixes { + supported[b] = true + } +} + +func asciiIsAlpha(b byte) bool { + return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') || ('0' <= b && b <= '9') +} diff --git a/format_test.go b/format_test.go new file mode 100644 index 0000000000000000000000000000000000000000..7bbefe9a58197940269056364be5638e2af69ed2 --- /dev/null +++ b/format_test.go @@ -0,0 +1,106 @@ +package cidutil + +import ( + "fmt" + "testing" + + c "github.com/ipfs/go-cid" + mb "github.com/multiformats/go-multibase" +) + +func TestFmt(t *testing.T) { + cids := map[string]string{ + "cidv0": "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn", + "cidv1": "zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD", + } + tests := []struct { + cidId string + newBase mb.Encoding + fmtStr string + result string + }{ + {"cidv0", -1, "%P", "cidv0-protobuf-sha2-256-32"}, + {"cidv0", -1, "%b-%v-%c-%h-%L", "base58btc-cidv0-protobuf-sha2-256-32"}, + {"cidv0", -1, "%s", "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"}, + {"cidv0", -1, "%S", "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"}, + {"cidv0", -1, "ver#%V/#%C/#%H/%L", "ver#0/#112/#18/32"}, + {"cidv0", -1, "%m", "zQmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"}, + {"cidv0", -1, "%M", "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"}, + {"cidv0", -1, "%d", "z72gdmFAgRzYHkJzKiL8MgMMRW3BTSCGyDHroPxJbxMJn"}, + {"cidv0", -1, "%D", "72gdmFAgRzYHkJzKiL8MgMMRW3BTSCGyDHroPxJbxMJn"}, + {"cidv0", 'B', "%S", "CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y"}, + {"cidv0", 'B', "%B%S", "BCIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y"}, + {"cidv1", -1, "%P", "cidv1-protobuf-sha2-256-32"}, + {"cidv1", -1, "%b-%v-%c-%h-%L", "base58btc-cidv1-protobuf-sha2-256-32"}, + {"cidv1", -1, "%s", "zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD"}, + {"cidv1", -1, "%S", "dj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD"}, + {"cidv1", -1, "ver#%V/#%C/#%H/%L", "ver#1/#112/#18/32"}, + {"cidv1", -1, "%m", "zQmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H"}, + {"cidv1", -1, "%M", "QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H"}, + {"cidv1", -1, "%d", "zAux4gVVsLRMXtsZ9fd3tFEZN4jGYB6kP37fgoZNTc11H"}, + {"cidv1", -1, "%D", "Aux4gVVsLRMXtsZ9fd3tFEZN4jGYB6kP37fgoZNTc11H"}, + {"cidv1", 'B', "%s", "BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA"}, + {"cidv1", 'B', "%S", "AFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA"}, + {"cidv1", 'B', "%B%S", "BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA"}, + } + for _, tc := range tests { + name := fmt.Sprintf("%s/%s", tc.cidId, tc.fmtStr) + if tc.newBase != -1 { + name = fmt.Sprintf("%s/%c", name, tc.newBase) + } + cidStr := cids[tc.cidId] + t.Run(name, func(t *testing.T) { + testFmt(t, cidStr, tc.newBase, tc.fmtStr, tc.result) + }) + } +} + +func testFmt(t *testing.T, cidStr string, newBase mb.Encoding, fmtStr string, result string) { + cid, err := c.Decode(cidStr) + if err != nil { + t.Fatal(err) + } + base := newBase + if newBase == -1 { + base, _ = c.ExtractEncoding(cidStr) + } + str, err := Format(fmtStr, base, cid) + if err != nil { + t.Fatal(err) + } + if str != result { + t.Error(fmt.Sprintf("expected: %s; but got: %s", result, str)) + } +} + +func TestScanForCid(t *testing.T) { + testStr := []byte(` +/ipfs/QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H 22 45 +/ipfs/zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD/foobar +BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA +skip me (too long): QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H876 +skip me (too short): bafybeietjgsrl3eqpqpcabv3g6iubytsifvq +bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja +bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja. +`) + cids := []string{ + "QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H", + "zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD", + "BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA", + "bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja", + "bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja", + } + + buf := testStr + idx := 0 + offset := 0 + for len(buf) > 0 { + _, j, _, cidStr := ScanForCid(buf) + if cidStr != "" && cids[idx] != cidStr { + t.Fatalf("Scan failed, expected %s, got %s (idx=%d offset=%d)", cids[idx], cidStr, idx, offset) + } + buf = buf[j:] + offset += j + idx++ + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..310b89f25a2133cf2626bd5dcf59e383e3a85d2b --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/ipfs/go-cidutil + +require ( + github.com/ipfs/go-cid v0.0.5 + github.com/multiformats/go-multibase v0.0.1 + github.com/multiformats/go-multihash v0.0.13 +) + +go 1.12 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..00bed8571139702272cc61b4637a18d96334fbc5 --- /dev/null +++ b/go.sum @@ -0,0 +1,54 @@ +github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyFSs7UnsU= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/ipfs/go-cid v0.0.2 h1:tuuKaZPU1M6HcejsO3AcYWW8sZ8MTvyxfc4uqB4eFE8= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= +github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16 h1:5W7KhL8HVF3XCFOweFD3BNESdnO8ewyYTFT2R+/b8FQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multihash v0.0.1 h1:HHwN1K12I+XllBCrqKnhX949Orn4oawPkegHMu2vDqQ= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.8 h1:wrYcW5yxSi3dU07n5jnuS5PrNwyHy0zRHGVoUugWvXg= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d h1:Z0Ahzd7HltpJtjAHHxX8QFP3j1yYgiuvjbjRzDj/KH0= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/inline.go b/inline.go new file mode 100644 index 0000000000000000000000000000000000000000..d140e1145dcd9a4b704315dbff1aa37fa7c747ad --- /dev/null +++ b/inline.go @@ -0,0 +1,26 @@ +package cidutil + +import ( + cid "github.com/ipfs/go-cid" + mhash "github.com/multiformats/go-multihash" +) + +// InlineBuilder is a cid.Builder that will use the id multihash when the +// size of the content is no more than limit +type InlineBuilder struct { + cid.Builder // Parent Builder + Limit int // Limit (inclusive) +} + +// WithCodec implements the cid.Builder interface +func (p InlineBuilder) WithCodec(c uint64) cid.Builder { + return InlineBuilder{p.Builder.WithCodec(c), p.Limit} +} + +// Sum implements the cid.Builder interface +func (p InlineBuilder) Sum(data []byte) (cid.Cid, error) { + if len(data) > p.Limit { + return p.Builder.Sum(data) + } + return cid.V1Builder{Codec: p.GetCodec(), MhType: mhash.ID}.Sum(data) +} diff --git a/inline_test.go b/inline_test.go new file mode 100644 index 0000000000000000000000000000000000000000..99bad7d3d6b7a087c522079020991d660f44e5b0 --- /dev/null +++ b/inline_test.go @@ -0,0 +1,33 @@ +package cidutil + +import ( + "math/rand" + "testing" + + cid "github.com/ipfs/go-cid" + mhash "github.com/multiformats/go-multihash" +) + +func TestInlineBuilderSmallValue(t *testing.T) { + builder := InlineBuilder{cid.V0Builder{}, 64} + c, err := builder.Sum([]byte("Hello World")) + if err != nil { + t.Fatal(err) + } + if c.Prefix().MhType != mhash.ID { + t.Fatal("Inliner builder failed to use ID Multihash on small values") + } +} + +func TestInlinerBuilderLargeValue(t *testing.T) { + builder := InlineBuilder{cid.V0Builder{}, 64} + data := make([]byte, 512) + rand.Read(data) + c, err := builder.Sum(data) + if err != nil { + t.Fatal(err) + } + if c.Prefix().MhType == mhash.ID { + t.Fatal("Inliner builder used ID Multihash on large values") + } +} diff --git a/set.go b/set.go new file mode 100644 index 0000000000000000000000000000000000000000..ea812a13dd96727f08106a43bc4709e5da9a0898 --- /dev/null +++ b/set.go @@ -0,0 +1,42 @@ +package cidutil + +import ( + "context" + + c "github.com/ipfs/go-cid" +) + +type Set = c.Set + +func NewSet() *Set { return c.NewSet() } + +// StreamingSet is an extension of Set which allows to implement back-pressure +// for the Visit function +type StreamingSet struct { + Set *Set + New chan c.Cid +} + +// NewStreamingSet initializes and returns new Set. +func NewStreamingSet() *StreamingSet { + return &StreamingSet{ + Set: c.NewSet(), + New: make(chan c.Cid), + } +} + +// Visitor creates new visitor which adds a Cids to the set and emits them to +// the set.New channel +func (s *StreamingSet) Visitor(ctx context.Context) func(c c.Cid) bool { + return func(c c.Cid) bool { + if s.Set.Visit(c) { + select { + case s.New <- c: + case <-ctx.Done(): + } + return true + } + + return false + } +} diff --git a/slice.go b/slice.go new file mode 100644 index 0000000000000000000000000000000000000000..fa39b64dee54c31a40c1541008f4a6d4f19809e6 --- /dev/null +++ b/slice.go @@ -0,0 +1,30 @@ +package cidutil + +import ( + "github.com/ipfs/go-cid" + "sort" +) + +// Slice is a convenience type for sorting CIDs +type Slice []cid.Cid + +func (s Slice) Len() int { + return len(s) +} + +func (s Slice) Less(i, j int) bool { + return s[i].KeyString() < s[j].KeyString() +} + +func (s Slice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s Slice) Sort() { + sort.Sort(s) +} + +// Sort sorts a slice of CIDs +func Sort(s []cid.Cid) { + Slice(s).Sort() +}