Merge pull request #53 from ipfs/fix/handle-overflow

Fix/handle overflow

hamt/hamt.go

@@ -203,9 +203,9 @@ func (ds *Shard) makeShardValue(lnk *ipld.Link) (*Shard, error) {
 }
 
 func hash(val []byte) []byte {
-	h := murmur3.New64()
+	h := murmur3.New128()
 	h.Write(val)
-	return h.Sum(nil)
+	return h.Sum(make([]byte, 0, 128/8))
 }
 
 // Set sets 'name' = nd in the HAMT
@@ -375,7 +375,10 @@ func (ds *Shard) rmChild(i int) error {
 }
 
 func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*Shard) error) error {
-	idx := hv.Next(ds.tableSizeLg2)
+	idx, err := hv.Next(ds.tableSizeLg2)
+	if err != nil {
+		return err
+	}
 	if ds.bitfield.Bit(int(idx)) {
 		cindex := ds.indexForBitPos(idx)
 
@@ -516,7 +519,10 @@ func (ds *Shard) walkTrie(ctx context.Context, cb func(*Shard) error) error {
 }
 
 func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error {
-	idx := hv.Next(ds.tableSizeLg2)
+	idx, err := hv.Next(ds.tableSizeLg2)
+	if err != nil {
+		return err
+	}
 	if !ds.bitfield.Bit(idx) {
 		return ds.insertChild(idx, key, val)
 	}

hamt/util.go

@@ -15,8 +15,16 @@ func mkmask(n int) byte {
 	return (1 << uint(n)) - 1
 }
 
-// Next returns the next 'i' bits of the hashBits value as an integer
-func (hb *hashBits) Next(i int) int {
+// Next returns the next 'i' bits of the hashBits value as an integer, or an
+// error if there aren't enough bits.
+func (hb *hashBits) Next(i int) (int, error) {
+	if hb.consumed+i > len(hb.b)*8 {
+		return 0, fmt.Errorf("sharded directory too deep")
+	}
+	return hb.next(i), nil
+}
+
+func (hb *hashBits) next(i int) int {
 	curbi := hb.consumed / 8
 	leftb := 8 - (hb.consumed % 8)
 
@@ -35,7 +43,7 @@ func (hb *hashBits) Next(i int) int {
 		out := int(mkmask(leftb) & curb)
 		out <<= uint(i - leftb)
 		hb.consumed += leftb
-		out += hb.Next(i - leftb)
+		out += hb.next(i - leftb)
 		return out
 	}
 }

hamt/util_test.go

@@ -9,37 +9,56 @@ func TestHashBitsEvenSizes(t *testing.T) {
 	hb := hashBits{b: buf}
 
 	for _, v := range buf {
-		if hb.Next(8) != int(v) {
-			t.Fatal("got wrong numbers back")
+		if a, _ := hb.Next(8); a != int(v) {
+			t.Fatalf("got wrong numbers back: expected %d, got %d", v, a)
 		}
 	}
 }
 
+func TestHashBitsOverflow(t *testing.T) {
+	buf := []byte{255}
+	hb := hashBits{b: buf}
+
+	for i := 0; i < 8; i++ {
+		bit, err := hb.Next(1)
+		if err != nil {
+			t.Fatalf("got %d bits back, expected 8: %s", i, err)
+		}
+		if bit != 1 {
+			t.Fatal("expected all one bits")
+		}
+	}
+	_, err := hb.Next(1)
+	if err == nil {
+		t.Error("overflowed the bit vector")
+	}
+}
+
 func TestHashBitsUneven(t *testing.T) {
 	buf := []byte{255, 127, 79, 45, 116, 99, 35, 17}
 	hb := hashBits{b: buf}
 
-	v := hb.Next(4)
+	v, _ := hb.Next(4)
 	if v != 15 {
 		t.Fatal("should have gotten 15: ", v)
 	}
 
-	v = hb.Next(4)
+	v, _ = hb.Next(4)
 	if v != 15 {
 		t.Fatal("should have gotten 15: ", v)
 	}
 
-	if v := hb.Next(3); v != 3 {
+	if v, _ := hb.Next(3); v != 3 {
 		t.Fatalf("expected 3, but got %b", v)
 	}
-	if v := hb.Next(3); v != 7 {
+	if v, _ := hb.Next(3); v != 7 {
 		t.Fatalf("expected 7, but got %b", v)
 	}
-	if v := hb.Next(3); v != 6 {
+	if v, _ := hb.Next(3); v != 6 {
 		t.Fatalf("expected 6, but got %b", v)
 	}
 
-	if v := hb.Next(15); v != 20269 {
+	if v, _ := hb.Next(15); v != 20269 {
 		t.Fatalf("expected 20269, but got %b (%d)", v, v)
 	}
 }