From 45b8e9c8ddd61131280ecf084649d03060e88be6 Mon Sep 17 00:00:00 2001 From: Eric Myhre Date: Fri, 12 Mar 2021 14:27:30 +0100 Subject: [PATCH] Update go-multihash; this upstreams much of the streaming hash work. --- go.mod | 5 +- go.sum | 24 +++++--- linking/cid/linksystem.go | 13 ++-- multihash/errata.go | 51 ---------------- multihash/multihash.go | 48 --------------- multihash/register/all/multihash_all.go | 23 ------- multihash/register/blake2/multihash_blake2.go | 48 --------------- .../register/murmur3/multihash_murmur3.go | 24 -------- multihash/register/sha3/multihash_sha3.go | 61 ------------------- 9 files changed, 24 insertions(+), 273 deletions(-) delete mode 100644 multihash/errata.go delete mode 100644 multihash/multihash.go delete mode 100644 multihash/register/all/multihash_all.go delete mode 100644 multihash/register/blake2/multihash_blake2.go delete mode 100644 multihash/register/murmur3/multihash_murmur3.go delete mode 100644 multihash/register/sha3/multihash_sha3.go diff --git a/go.mod b/go.mod index 8d64288..b1a1c34 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,8 @@ go 1.14 require ( github.com/ipfs/go-cid v0.0.4 - github.com/minio/sha256-simd v0.1.1 // indirect - github.com/mr-tron/base58 v1.1.3 // indirect + github.com/multiformats/go-multihash v0.0.15 github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a - golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect - golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 // indirect ) diff --git a/go.sum b/go.sum index bc17d60..60bb231 100644 --- a/go.sum +++ b/go.sum @@ -4,21 +4,27 @@ github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= -github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= +github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI= github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -31,13 +37,15 @@ github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/linking/cid/linksystem.go b/linking/cid/linksystem.go index 3e2cb3a..40f2b44 100644 --- a/linking/cid/linksystem.go +++ b/linking/cid/linksystem.go @@ -4,9 +4,10 @@ import ( "fmt" "hash" + "github.com/multiformats/go-multihash/core" + "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/multicodec" - "github.com/ipld/go-ipld-prime/multihash" ) func DefaultLinkSystem() ipld.LinkSystem { @@ -39,13 +40,13 @@ func DefaultLinkSystem() ipld.LinkSystem { HasherChooser: func(lp ipld.LinkPrototype) (hash.Hash, error) { switch lp2 := lp.(type) { case LinkPrototype: - fn, ok := multihash.Registry[lp2.MhType] - if !ok { - return nil, fmt.Errorf("no hasher registered for multihash indicator 0x%x", lp2.MhType) + h, err := multihash.GetHasher(lp2.MhType) + if err != nil { + return nil, fmt.Errorf("no hasher registered for multihash indicator 0x%x: %w", lp2.MhType, err) } - return fn(), nil + return h, nil default: - return nil, fmt.Errorf("this decoderChooser can only handle cidlink.LinkPrototype; got %T", lp) + return nil, fmt.Errorf("this hasherChooser can only handle cidlink.LinkPrototype; got %T", lp) } }, } diff --git a/multihash/errata.go b/multihash/errata.go deleted file mode 100644 index 7c74321..0000000 --- a/multihash/errata.go +++ /dev/null @@ -1,51 +0,0 @@ -package multihash - -import ( - "bytes" - "crypto/sha256" - "hash" -) - -type identityMultihash struct { - bytes.Buffer -} - -func (identityMultihash) BlockSize() int { - return 32 // A prefered block size is nonsense for the "identity" "hash". An arbitrary but unsurprising and positive nonzero number has been chosen to minimize the odds of fascinating bugs. -} - -func (x identityMultihash) Size() int { - return x.Len() -} - -func (x identityMultihash) Sum(digest []byte) []byte { - return x.Bytes() -} - -type doubleSha256 struct { - main hash.Hash -} - -func (x doubleSha256) Write(body []byte) (int, error) { - return x.main.Write(body) -} - -func (doubleSha256) BlockSize() int { - return sha256.BlockSize -} - -func (doubleSha256) Size() int { - return sha256.Size -} - -func (x doubleSha256) Reset() { - x.main.Reset() -} - -func (x doubleSha256) Sum(digest []byte) []byte { - intermediate := [sha256.Size]byte{} - x.main.Sum(intermediate[:]) - h2 := sha256.New() - h2.Write(intermediate[:]) - return h2.Sum(digest) -} diff --git a/multihash/multihash.go b/multihash/multihash.go deleted file mode 100644 index e783cb6..0000000 --- a/multihash/multihash.go +++ /dev/null @@ -1,48 +0,0 @@ -package multihash - -import ( - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "crypto/sha512" - "hash" -) - -// Registry is a simple map which maps a multihash indicator number -// to a standard golang Hash interface. -// -// Multihash indicator numbers are reserved and described in -// https://github.com/multiformats/multicodec/blob/master/table.csv . -// The keys used in this map must match those reservations. -// -// Hashers which are available in the golang stdlib are registered here automatically. -// -// Packages which want to register more hashing functions (and have a multihash number reserved!) -// are encouraged to do so at package init time. -// (Doing this at package init time ensures this map can be accessed without race conditions.) -// -// The linking/cid.DefaultLinkSystem will use this map to find hashers -// to use when serializing data and computing links, -// and when loading data from storage and verifying its integrity. -// -// This registry map is only used for default behaviors. -// If you don't want to rely on it, you can always construct your own LinkSystem. -// (For this reason, there's no special effort made to detect conflicting registrations in this map. -// If more than one package registers for the same multicodec indicator, and -// you somehow end up with both in your import tree, and yet care about which wins: -// then just don't use this registry anymore: make a LinkSystem that does what you need.) -// This should never be done to make behavior alterations -// (hash functions are well standardized and so is the multihash indicator table), -// but may be relevant if one is really itching to try out different hash implementations for performance reasons. -var Registry = make(map[uint64]func() hash.Hash) - -func init() { - Registry[0x00] = func() hash.Hash { return &identityMultihash{} } - Registry[0xd5] = md5.New - Registry[0x11] = sha1.New - Registry[0x12] = sha256.New - Registry[0x13] = sha512.New - // Registry[0x1f] = sha256.New224 // SOON - // Registry[0x20] = sha512.New384 // SOON - Registry[0x56] = func() hash.Hash { return &doubleSha256{} } -} diff --git a/multihash/register/all/multihash_all.go b/multihash/register/all/multihash_all.go deleted file mode 100644 index c52118f..0000000 --- a/multihash/register/all/multihash_all.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - This package has no purpose except to perform registration of mulithashes. - - It is meant to be used as a side-effecting import, e.g. - - import ( - _ "github.com/ipld/go-ipld-prime/mulithash/register/all" - ) - - This package registers many multihashes at once. - Importing it will increase the size of your dependency tree significantly. - It's recommended that you import this package if you're building some - kind of data broker application, which may need to handle many different kinds of hashes; - if you're building an application which you know only handles a specific hash, - importing this package may bloat your builds unnecessarily. -*/ -package all - -import ( - _ "github.com/ipld/go-ipld-prime/multihash/register/blake2" - _ "github.com/ipld/go-ipld-prime/multihash/register/murmur3" - _ "github.com/ipld/go-ipld-prime/multihash/register/sha3" -) diff --git a/multihash/register/blake2/multihash_blake2.go b/multihash/register/blake2/multihash_blake2.go deleted file mode 100644 index b56188b..0000000 --- a/multihash/register/blake2/multihash_blake2.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - This package has no purpose except to perform registration of multihashes. - - It is meant to be used as a side-effecting import, e.g. - - import ( - _ "github.com/ipld/go-ipld-prime/mulithash/register/blake2" - ) - - This package registers several multihashes for the blake2 family - (both the 's' and the 'b' variants, and in a variety of sizes). -*/ -package blake2 - -import ( - "hash" - - "github.com/minio/blake2b-simd" - "golang.org/x/crypto/blake2s" - - "github.com/ipld/go-ipld-prime/multihash" -) - -const ( - BLAKE2B_MIN = 0xb201 - BLAKE2B_MAX = 0xb240 - BLAKE2S_MIN = 0xb241 - BLAKE2S_MAX = 0xb260 -) - -func init() { - // BLAKE2S - // This package only enables support for 32byte (256 bit) blake2s. - multihash.Registry[BLAKE2S_MIN+31] = func() hash.Hash { h, _ := blake2s.New256(nil); return h } - - // BLAKE2B - // There's a whole range of these. - for c := uint64(BLAKE2B_MIN); c <= BLAKE2B_MAX; c++ { - size := int(c - BLAKE2B_MIN + 1) - multihash.Registry[c] = func() hash.Hash { - hasher, err := blake2b.New(&blake2b.Config{Size: uint8(size)}) - if err != nil { - panic(err) - } - return hasher - } - } -} diff --git a/multihash/register/murmur3/multihash_murmur3.go b/multihash/register/murmur3/multihash_murmur3.go deleted file mode 100644 index a4876d7..0000000 --- a/multihash/register/murmur3/multihash_murmur3.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - This package has no purpose except to perform registration of multihashes. - - It is meant to be used as a side-effecting import, e.g. - - import ( - _ "github.com/ipld/go-ipld-prime/mulithash/register/murmur3" - ) - - This package registers multihashes for the murmur3 family. -*/ -package murmur3 - -// import ( -// "github.com/gxed/hashland/murmur3" -// -// "github.com/ipld/go-ipld-prime/multihash" -// ) - -func init() { - // REVIEW: what go-multihash has done historically is New32, but this doesn't match what the multihash table says, which is 128! - // These are also very clearly noncryptographic functions and not suitable for content-addressing use (and would require writing adapters to qualify for hash.Hash), so I'm opting to... not. - // multihash.Registry[0x22] = murmur3.New32 -} diff --git a/multihash/register/sha3/multihash_sha3.go b/multihash/register/sha3/multihash_sha3.go deleted file mode 100644 index d1f1084..0000000 --- a/multihash/register/sha3/multihash_sha3.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - This package has no purpose except to perform registration of multihashes. - - It is meant to be used as a side-effecting import, e.g. - - import ( - _ "github.com/ipld/go-ipld-prime/mulithash/register/sha3" - ) - - This package registers several multihashes for the sha3 family. - This also includes some functions known as "shake" and "keccak", - since they share much of their implementation and come in the same repos. -*/ -package sha3 - -import ( - "hash" - - "golang.org/x/crypto/sha3" - - "github.com/ipld/go-ipld-prime/multihash" -) - -func init() { - multihash.Registry[0x14] = sha3.New512 - multihash.Registry[0x15] = sha3.New384 - multihash.Registry[0x16] = sha3.New256 - multihash.Registry[0x17] = sha3.New224 - multihash.Registry[0x18] = func() hash.Hash { return shakeNormalizer{sha3.NewShake128(), 128 / 8} } - multihash.Registry[0x19] = func() hash.Hash { return shakeNormalizer{sha3.NewShake256(), 256 / 8} } - multihash.Registry[0x1B] = sha3.NewLegacyKeccak256 - multihash.Registry[0x1D] = sha3.NewLegacyKeccak512 -} - -// sha3.ShakeHash presents a somewhat odd interface, and requires a wrapper to normalize it to the usual hash.Hash interface. -// -// Some of the fiddly bits required by this normalization probably makes it undesirable for use in the highest performance applications; -// There's at least one extra allocation in constructing it (sha3.ShakeHash is an interface, so that's one heap escape; and there's a second heap escape when this normalizer struct gets boxed into a hash.Hash interface), -// and there's at least one extra allocation in getting a sum out of it (because reading a shake hash is a mutation (!) and the API only provides cloning as a way to escape this). -// Fun. -type shakeNormalizer struct { - sha3.ShakeHash - size int -} - -func (shakeNormalizer) BlockSize() int { - return 32 // Shake doesn't have a prefered block size, apparently. An arbitrary but unsurprising and positive nonzero number has been chosen to minimize the odds of fascinating bugs. -} - -func (x shakeNormalizer) Size() int { - return x.size -} - -func (x shakeNormalizer) Sum(digest []byte) []byte { - if len(digest) != x.size { - digest = make([]byte, x.size) - } - h2 := x.Clone() // clone it, because reading mutates this kind of hash (!) which is not the standard contract for a Hash.Sum method. - h2.Read(digest) // not capable of underreading. See sha3.ShakeSum256 for similar usage. - return digest -} -- GitLab