Skip to content

GitLab

Unverified Commit ff794fa7 authored by Steven Allen's avatar Steven Allen Committed by GitHub
Browse files
Options

Merge pull request #2 from ipfs/fix/make-safe 

feat: make safe
parents 4492942c 93a5d89f
Hide whitespace changes
Inline Side-by-side
Showing with 157 additions and 37 deletions
bbloom.go
View file @ ff794fa7
......@@ -22,16 +22,14 @@ package bbloom
import (
"bytes"
"encoding/binary"
"encoding/json"
"errors"
"math"
"math/bits"
"sync"
"unsafe"
)
// helper
var mask = []uint8{1, 2, 4, 8, 16, 32, 64, 128}
func getSize(ui64 uint64) (size uint64, exponent uint64) {
if ui64 < uint64(512) {
ui64 = uint64(512)
......@@ -88,10 +86,8 @@ func NewWithBoolset(bs *[]byte, locs uint64) (bloomfilter *Bloom) {
if err != nil {
panic(err) // Should never happen
}
ptr := uintptr(unsafe.Pointer(&bloomfilter.bitset[0]))
for _, b := range *bs {
*(*uint8)(unsafe.Pointer(ptr)) = b
ptr++
for i := range bloomfilter.bitset {
bloomfilter.bitset[i] = binary.BigEndian.Uint64((*bs)[i<<3:])
}
return bloomfilter
}
......@@ -221,23 +217,21 @@ func (bl *Bloom) Clear() {
// Set
// set the bit[idx] of bitsit
func (bl *Bloom) set(idx uint64) {
ptr := unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[idx>>6])) + uintptr((idx%64)>>3))
*(*uint8)(ptr) |= mask[idx%8]
bl.bitset[idx>>6] |= 1 << (idx % 64)
}
func (bl *Bloom) getSet(idx uint64) bool {
ptr := unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[idx>>6])) + uintptr((idx%64)>>3))
res := *(*uint8)(ptr)&mask[idx%8] > 0
*(*uint8)(ptr) |= mask[idx%8]
return res
cur := bl.bitset[idx>>6]
bit := uint64(1 << (idx % 64))
bl.bitset[idx>>6] = cur | bit
return (cur & bit) > 0
}
// IsSet
// check if bit[idx] of bitset is set
// returns true/false
func (bl *Bloom) isSet(idx uint64) bool {
ptr := unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[idx>>6])) + uintptr((idx%64)>>3))
return *(*uint8)(ptr)&mask[idx%8] > 0
return bl.bitset[idx>>6]&(1<<(idx%64)) > 0
}
// JSONMarshal
......@@ -248,10 +242,8 @@ func (bl *Bloom) JSONMarshal() ([]byte, error) {
bloomImEx := bloomJSONImExport{}
bloomImEx.SetLocs = uint64(bl.setLocs)
bloomImEx.FilterSet = make([]byte, len(bl.bitset)<<3)
ptr := uintptr(unsafe.Pointer(&bl.bitset[0]))
for i := range bloomImEx.FilterSet {
bloomImEx.FilterSet[i] = *(*byte)(unsafe.Pointer(ptr))
ptr++
for i, w := range bl.bitset {
binary.BigEndian.PutUint64(bloomImEx.FilterSet[i<<3:], w)
}
data, err := json.Marshal(bloomImEx)
return data, err
......@@ -272,24 +264,11 @@ func JSONUnmarshal(dbData []byte) *Bloom {
func (bl *Bloom) FillRatio() float64 {
count := uint64(0)
for _, b := range bl.bitset {
count += uint64(popcount(b))
count += uint64(bits.OnesCount64(b))
}
return float64(count) / float64(bl.size+1)
}
func popcount(x uint64) uint {
const (
m1 = 0x5555555555555555 //binary: 0101...
m2 = 0x3333333333333333 //binary: 00110011..
m4 = 0x0f0f0f0f0f0f0f0f //binary: 4 zeros, 4 ones ...
h01 = 0x0101010101010101 //the sum of 256 to the power of 0,1,2,3...
)
x -= (x >> 1) & m1 //put count of each 2 bits into those 2 bits
x = (x & m2) + ((x >> 2) & m2) //put count of each 4 bits into those 4 bits
x = (x + (x >> 4)) & m4 //put count of each 8 bits into those 8 bits
return uint((x * h01) >> 56)
}
// // alternative hashFn
// func (bl Bloom) fnv64a(b *[]byte) (l, h uint64) {
// h64 := fnv.New64a()
......
bbloom_test.go
View file @ ff794fa7
......@@ -100,7 +100,7 @@ func TestFillRatio(t *testing.T) {
}
}
func ExampleM_NewAddHasAddIfNotHas() {
func ExampleBloom_AddIfNotHas() {
bf, err := New(float64(512), float64(1))
if err != nil {
panic(err)
......
sipHash.go
View file @ ff794fa7
......@@ -12,7 +12,7 @@ package bbloom
// Hash returns the 64-bit SipHash-2-4 of the given byte slice with two 64-bit
// parts of 128-bit key: k0 and k1.
func (bl Bloom) sipHash(p []byte) (l, h uint64) {
func (bl *Bloom) sipHash(p []byte) (l, h uint64) {
// Initialization.
v0 := uint64(8317987320269560794) // k0 ^ 0x736f6d6570736575
v1 := uint64(7237128889637516672) // k1 ^ 0x646f72616e646f6d
......
words.txt 0 → 100644
View file @ ff794fa7
2014/01/01 00:00:00 /info.html
2014/01/01 00:00:00 /info.html
2014/01/01 00:00:01 /info.html
2014/01/01 00:00:02 /info.html
2014/01/01 00:00:03 /info.html
2014/01/01 00:00:04 /info.html
2014/01/01 00:00:05 /info.html
2014/01/01 00:00:06 /info.html
2014/01/01 00:00:07 /info.html
2014/01/01 00:00:08 /info.html
2014/01/01 00:00:09 /info.html
2014/01/01 00:00:10 /info.html
2014/01/01 00:00:11 /info.html
2014/01/01 00:00:12 /info.html
2014/01/01 00:00:13 /info.html
2014/01/01 00:00:14 /info.html
2014/01/01 00:00:15 /info.html
2014/01/01 00:00:16 /info.html
2014/01/01 00:00:17 /info.html
2014/01/01 00:00:18 /info.html
2014/01/01 00:00:19 /info.html
2014/01/01 00:00:20 /info.html
2014/01/01 00:00:21 /info.html
2014/01/01 00:00:22 /info.html
2014/01/01 00:00:23 /info.html
2014/01/01 00:00:24 /info.html
2014/01/01 00:00:25 /info.html
2014/01/01 00:00:26 /info.html
2014/01/01 00:00:27 /info.html
2014/01/01 00:00:28 /info.html
2014/01/01 00:00:29 /info.html
2014/01/01 00:00:30 /info.html
2014/01/01 00:00:31 /info.html
2014/01/01 00:00:32 /info.html
2014/01/01 00:00:33 /info.html
2014/01/01 00:00:34 /info.html
2014/01/01 00:00:35 /info.html
2014/01/01 00:00:36 /info.html
2014/01/01 00:00:37 /info.html
2014/01/01 00:00:38 /info.html
2014/01/01 00:00:39 /info.html
2014/01/01 00:00:40 /info.html
2014/01/01 00:00:41 /info.html
2014/01/01 00:00:42 /info.html
2014/01/01 00:00:43 /info.html
2014/01/01 00:00:44 /info.html
2014/01/01 00:00:45 /info.html
2014/01/01 00:00:46 /info.html
2014/01/01 00:00:47 /info.html
2014/01/01 00:00:48 /info.html
2014/01/01 00:00:49 /info.html
2014/01/01 00:00:50 /info.html
2014/01/01 00:00:51 /info.html
2014/01/01 00:00:52 /info.html
2014/01/01 00:00:53 /info.html
2014/01/01 00:00:54 /info.html
2014/01/01 00:00:55 /info.html
2014/01/01 00:00:56 /info.html
2014/01/01 00:00:57 /info.html
2014/01/01 00:00:58 /info.html
2014/01/01 00:00:59 /info.html
2014/01/01 00:01:00 /info.html
2014/01/01 00:01:01 /info.html
2014/01/01 00:01:02 /info.html
2014/01/01 00:01:03 /info.html
2014/01/01 00:01:04 /info.html
2014/01/01 00:01:05 /info.html
2014/01/01 00:01:06 /info.html
2014/01/01 00:01:07 /info.html
2014/01/01 00:01:08 /info.html
2014/01/01 00:01:09 /info.html
2014/01/01 00:01:10 /info.html
2014/01/01 00:01:11 /info.html
2014/01/01 00:01:12 /info.html
2014/01/01 00:01:13 /info.html
2014/01/01 00:01:14 /info.html
2014/01/01 00:01:15 /info.html
2014/01/01 00:01:16 /info.html
2014/01/01 00:01:17 /info.html
2014/01/01 00:01:18 /info.html
2014/01/01 00:01:19 /info.html
2014/01/01 00:01:20 /info.html
2014/01/01 00:01:21 /info.html
2014/01/01 00:01:22 /info.html
2014/01/01 00:01:23 /info.html
2014/01/01 00:01:24 /info.html
2014/01/01 00:01:25 /info.html
2014/01/01 00:01:26 /info.html
2014/01/01 00:01:27 /info.html
2014/01/01 00:01:28 /info.html
2014/01/01 00:01:29 /info.html
2014/01/01 00:01:30 /info.html
2014/01/01 00:01:31 /info.html
2014/01/01 00:01:32 /info.html
2014/01/01 00:01:33 /info.html
2014/01/01 00:01:34 /info.html
2014/01/01 00:01:35 /info.html
2014/01/01 00:01:36 /info.html
2014/01/01 00:01:37 /info.html
2014/01/01 00:01:38 /info.html
2014/01/01 00:01:39 /info.html
2014/01/01 00:01:40 /info.html
2014/01/01 00:01:41 /info.html
2014/01/01 00:01:42 /info.html
2014/01/01 00:01:43 /info.html
2014/01/01 00:01:44 /info.html
2014/01/01 00:01:45 /info.html
2014/01/01 00:01:46 /info.html
2014/01/01 00:01:47 /info.html
2014/01/01 00:01:48 /info.html
2014/01/01 00:01:49 /info.html
2014/01/01 00:01:50 /info.html
2014/01/01 00:01:51 /info.html
2014/01/01 00:01:52 /info.html
2014/01/01 00:01:53 /info.html
2014/01/01 00:01:54 /info.html
2014/01/01 00:01:55 /info.html
2014/01/01 00:01:56 /info.html
2014/01/01 00:01:57 /info.html
2014/01/01 00:01:58 /info.html
2014/01/01 00:01:59 /info.html
2014/01/01 00:02:00 /info.html
2014/01/01 00:02:01 /info.html
2014/01/01 00:02:02 /info.html
2014/01/01 00:02:03 /info.html
2014/01/01 00:02:04 /info.html
2014/01/01 00:02:05 /info.html
2014/01/01 00:02:06 /info.html
2014/01/01 00:02:07 /info.html
2014/01/01 00:02:08 /info.html
2014/01/01 00:02:09 /info.html
2014/01/01 00:02:10 /info.html
2014/01/01 00:02:11 /info.html
2014/01/01 00:02:12 /info.html
2014/01/01 00:02:13 /info.html
2014/01/01 00:02:14 /info.html
2014/01/01 00:02:15 /info.html
2014/01/01 00:02:16 /info.html
2014/01/01 00:02:17 /info.html
2014/01/01 00:02:18 /info.html
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment