Skip to content

GitLab

Commit 45b8e9c8 authored by Eric Myhre's avatar Eric Myhre
Browse files
Options

Update go-multihash; this upstreams much of the streaming hash work.

parent 8fef5312
Hide whitespace changes
Inline Side-by-side
Showing with 24 additions and 273 deletions
go.mod
View file @ 45b8e9c8
......@@ -4,11 +4,8 @@ go 1.14
require (
github.com/ipfs/go-cid v0.0.4
github.com/minio/sha256-simd v0.1.1 // indirect
github.com/mr-tron/base58 v1.1.3 // indirect
github.com/multiformats/go-multihash v0.0.15
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 // indirect
)
go.sum
View file @ 45b8e9c8
......@@ -4,21 +4,27 @@ github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8=
github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA=
github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM=
github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM=
github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI=
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
......@@ -31,13 +37,15 @@ github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg=
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
linking/cid/linksystem.go
View file @ 45b8e9c8
......@@ -4,9 +4,10 @@ import (
"fmt"
"hash"
"github.com/multiformats/go-multihash/core"
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/multicodec"
"github.com/ipld/go-ipld-prime/multihash"
)
func DefaultLinkSystem() ipld.LinkSystem {
......@@ -39,13 +40,13 @@ func DefaultLinkSystem() ipld.LinkSystem {
HasherChooser: func(lp ipld.LinkPrototype) (hash.Hash, error) {
switch lp2 := lp.(type) {
case LinkPrototype:
fn, ok := multihash.Registry[lp2.MhType]
if !ok {
return nil, fmt.Errorf("no hasher registered for multihash indicator 0x%x", lp2.MhType)
h, err := multihash.GetHasher(lp2.MhType)
if err != nil {
return nil, fmt.Errorf("no hasher registered for multihash indicator 0x%x: %w", lp2.MhType, err)
}
return fn(), nil
return h, nil
default:
return nil, fmt.Errorf("this decoderChooser can only handle cidlink.LinkPrototype; got %T", lp)
return nil, fmt.Errorf("this hasherChooser can only handle cidlink.LinkPrototype; got %T", lp)
}
},
}
......
multihash/errata.go deleted 100644 → 0
View file @ 8fef5312
package multihash
import (
"bytes"
"crypto/sha256"
"hash"
)
type identityMultihash struct {
bytes.Buffer
}
func (identityMultihash) BlockSize() int {
return 32 // A prefered block size is nonsense for the "identity" "hash". An arbitrary but unsurprising and positive nonzero number has been chosen to minimize the odds of fascinating bugs.
}
func (x identityMultihash) Size() int {
return x.Len()
}
func (x identityMultihash) Sum(digest []byte) []byte {
return x.Bytes()
}
type doubleSha256 struct {
main hash.Hash
}
func (x doubleSha256) Write(body []byte) (int, error) {
return x.main.Write(body)
}
func (doubleSha256) BlockSize() int {
return sha256.BlockSize
}
func (doubleSha256) Size() int {
return sha256.Size
}
func (x doubleSha256) Reset() {
x.main.Reset()
}
func (x doubleSha256) Sum(digest []byte) []byte {
intermediate := [sha256.Size]byte{}
x.main.Sum(intermediate[:])
h2 := sha256.New()
h2.Write(intermediate[:])
return h2.Sum(digest)
}
multihash/multihash.go deleted 100644 → 0
View file @ 8fef5312
package multihash
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"hash"
)
// Registry is a simple map which maps a multihash indicator number
// to a standard golang Hash interface.
//
// Multihash indicator numbers are reserved and described in
// https://github.com/multiformats/multicodec/blob/master/table.csv .
// The keys used in this map must match those reservations.
//
// Hashers which are available in the golang stdlib are registered here automatically.
//
// Packages which want to register more hashing functions (and have a multihash number reserved!)
// are encouraged to do so at package init time.
// (Doing this at package init time ensures this map can be accessed without race conditions.)
//
// The linking/cid.DefaultLinkSystem will use this map to find hashers
// to use when serializing data and computing links,
// and when loading data from storage and verifying its integrity.
//
// This registry map is only used for default behaviors.
// If you don't want to rely on it, you can always construct your own LinkSystem.
// (For this reason, there's no special effort made to detect conflicting registrations in this map.
// If more than one package registers for the same multicodec indicator, and
// you somehow end up with both in your import tree, and yet care about which wins:
// then just don't use this registry anymore: make a LinkSystem that does what you need.)
// This should never be done to make behavior alterations
// (hash functions are well standardized and so is the multihash indicator table),
// but may be relevant if one is really itching to try out different hash implementations for performance reasons.
var Registry = make(map[uint64]func() hash.Hash)
func init() {
Registry[0x00] = func() hash.Hash { return &identityMultihash{} }
Registry[0xd5] = md5.New
Registry[0x11] = sha1.New
Registry[0x12] = sha256.New
Registry[0x13] = sha512.New
// Registry[0x1f] = sha256.New224 // SOON
// Registry[0x20] = sha512.New384 // SOON
Registry[0x56] = func() hash.Hash { return &doubleSha256{} }
}
multihash/register/all/multihash_all.go deleted 100644 → 0
View file @ 8fef5312
/*
This package has no purpose except to perform registration of mulithashes.
It is meant to be used as a side-effecting import, e.g.
import (
_ "github.com/ipld/go-ipld-prime/mulithash/register/all"
)
This package registers many multihashes at once.
Importing it will increase the size of your dependency tree significantly.
It's recommended that you import this package if you're building some
kind of data broker application, which may need to handle many different kinds of hashes;
if you're building an application which you know only handles a specific hash,
importing this package may bloat your builds unnecessarily.
*/
package all
import (
_ "github.com/ipld/go-ipld-prime/multihash/register/blake2"
_ "github.com/ipld/go-ipld-prime/multihash/register/murmur3"
_ "github.com/ipld/go-ipld-prime/multihash/register/sha3"
)
multihash/register/blake2/multihash_blake2.go deleted 100644 → 0
View file @ 8fef5312
/*
This package has no purpose except to perform registration of multihashes.
It is meant to be used as a side-effecting import, e.g.
import (
_ "github.com/ipld/go-ipld-prime/mulithash/register/blake2"
)
This package registers several multihashes for the blake2 family
(both the 's' and the 'b' variants, and in a variety of sizes).
*/
package blake2
import (
"hash"
"github.com/minio/blake2b-simd"
"golang.org/x/crypto/blake2s"
"github.com/ipld/go-ipld-prime/multihash"
)
const (
BLAKE2B_MIN = 0xb201
BLAKE2B_MAX = 0xb240
BLAKE2S_MIN = 0xb241
BLAKE2S_MAX = 0xb260
)
func init() {
// BLAKE2S
// This package only enables support for 32byte (256 bit) blake2s.
multihash.Registry[BLAKE2S_MIN+31] = func() hash.Hash { h, _ := blake2s.New256(nil); return h }
// BLAKE2B
// There's a whole range of these.
for c := uint64(BLAKE2B_MIN); c <= BLAKE2B_MAX; c++ {
size := int(c - BLAKE2B_MIN + 1)
multihash.Registry[c] = func() hash.Hash {
hasher, err := blake2b.New(&blake2b.Config{Size: uint8(size)})
if err != nil {
panic(err)
}
return hasher
}
}
}
multihash/register/murmur3/multihash_murmur3.go deleted 100644 → 0
View file @ 8fef5312
/*
This package has no purpose except to perform registration of multihashes.
It is meant to be used as a side-effecting import, e.g.
import (
_ "github.com/ipld/go-ipld-prime/mulithash/register/murmur3"
)
This package registers multihashes for the murmur3 family.
*/
package murmur3
// import (
// "github.com/gxed/hashland/murmur3"
//
// "github.com/ipld/go-ipld-prime/multihash"
// )
func init() {
// REVIEW: what go-multihash has done historically is New32, but this doesn't match what the multihash table says, which is 128!
// These are also very clearly noncryptographic functions and not suitable for content-addressing use (and would require writing adapters to qualify for hash.Hash), so I'm opting to... not.
// multihash.Registry[0x22] = murmur3.New32
}
multihash/register/sha3/multihash_sha3.go deleted 100644 → 0
View file @ 8fef5312
/*
This package has no purpose except to perform registration of multihashes.
It is meant to be used as a side-effecting import, e.g.
import (
_ "github.com/ipld/go-ipld-prime/mulithash/register/sha3"
)
This package registers several multihashes for the sha3 family.
This also includes some functions known as "shake" and "keccak",
since they share much of their implementation and come in the same repos.
*/
package sha3
import (
"hash"
"golang.org/x/crypto/sha3"
"github.com/ipld/go-ipld-prime/multihash"
)
func init() {
multihash.Registry[0x14] = sha3.New512
multihash.Registry[0x15] = sha3.New384
multihash.Registry[0x16] = sha3.New256
multihash.Registry[0x17] = sha3.New224
multihash.Registry[0x18] = func() hash.Hash { return shakeNormalizer{sha3.NewShake128(), 128 / 8} }
multihash.Registry[0x19] = func() hash.Hash { return shakeNormalizer{sha3.NewShake256(), 256 / 8} }
multihash.Registry[0x1B] = sha3.NewLegacyKeccak256
multihash.Registry[0x1D] = sha3.NewLegacyKeccak512
}
// sha3.ShakeHash presents a somewhat odd interface, and requires a wrapper to normalize it to the usual hash.Hash interface.
//
// Some of the fiddly bits required by this normalization probably makes it undesirable for use in the highest performance applications;
// There's at least one extra allocation in constructing it (sha3.ShakeHash is an interface, so that's one heap escape; and there's a second heap escape when this normalizer struct gets boxed into a hash.Hash interface),
// and there's at least one extra allocation in getting a sum out of it (because reading a shake hash is a mutation (!) and the API only provides cloning as a way to escape this).
// Fun.
type shakeNormalizer struct {
sha3.ShakeHash
size int
}
func (shakeNormalizer) BlockSize() int {
return 32 // Shake doesn't have a prefered block size, apparently. An arbitrary but unsurprising and positive nonzero number has been chosen to minimize the odds of fascinating bugs.
}
func (x shakeNormalizer) Size() int {
return x.size
}
func (x shakeNormalizer) Sum(digest []byte) []byte {
if len(digest) != x.size {
digest = make([]byte, x.size)
}
h2 := x.Clone() // clone it, because reading mutates this kind of hash (!) which is not the standard contract for a Hash.Sum method.
h2.Read(digest) // not capable of underreading. See sha3.ShakeSum256 for similar usage.
return digest
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment