implement unpin and add a datastore backed blockset

blocks/bloom/filter.go

 package bloom
 
 import (
+	"errors"
 	"fmt"
 	"hash"
 	"hash/adler32"
@@ -12,6 +13,7 @@ import (
 type Filter interface {
 	Add([]byte)
 	Find([]byte) bool
+	Merge(Filter) (Filter, error)
 }
 
 func BasicFilter() Filter {
@@ -68,3 +70,26 @@ func bytesMod(b []byte, modulo int64) int64 {
 
 	return result.Int64()
 }
+
+func (f *filter) Merge(o Filter) (Filter, error) {
+	casfil, ok := o.(*filter)
+	if !ok {
+		return nil, errors.New("Unsupported filter type")
+	}
+
+	if len(casfil.filter) != len(f.filter) {
+		return nil, errors.New("filter lengths must match!")
+	}
+
+	nfilt := new(filter)
+
+	// this bit is sketchy, need a way of comparing hash functions
+	nfilt.hashes = f.hashes
+
+	nfilt.filter = make([]byte, len(f.filter))
+	for i, v := range f.filter {
+		nfilt.filter[i] = v | casfil.filter[i]
+	}
+
+	return nfilt, nil
+}

blocks/set/dbset.go

+package set
+
+import (
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	"github.com/jbenet/go-ipfs/blocks/bloom"
+	"github.com/jbenet/go-ipfs/util"
+)
+
+type datastoreBlockSet struct {
+	dstore ds.Datastore
+	bset   BlockSet
+	prefix string
+}
+
+func NewDBWrapperSet(d ds.Datastore, prefix string, bset BlockSet) BlockSet {
+	return &datastoreBlockSet{
+		dstore: d,
+		bset:   bset,
+		prefix: prefix,
+	}
+}
+
+func (d *datastoreBlockSet) AddBlock(k util.Key) {
+	err := d.dstore.Put(d.prefixKey(k), []byte{})
+	if err != nil {
+		log.Error("blockset put error: %s", err)
+	}
+
+	d.bset.AddBlock(k)
+}
+
+func (d *datastoreBlockSet) RemoveBlock(k util.Key) {
+	d.bset.RemoveBlock(k)
+	if !d.bset.HasKey(k) {
+		d.dstore.Delete(d.prefixKey(k))
+	}
+}
+
+func (d *datastoreBlockSet) HasKey(k util.Key) bool {
+	return d.bset.HasKey(k)
+}
+
+func (d *datastoreBlockSet) GetBloomFilter() bloom.Filter {
+	return d.bset.GetBloomFilter()
+}
+
+func (d *datastoreBlockSet) prefixKey(k util.Key) ds.Key {
+	return (util.Key(d.prefix) + k).DsKey()
+}

pin/refset.go → blocks/set/refset.go

-package pin
+package set
 
 import (
 	"github.com/jbenet/go-ipfs/blocks/bloom"
-	"github.com/jbenet/go-ipfs/blocks/set"
 	"github.com/jbenet/go-ipfs/util"
 )
 
@@ -10,7 +9,7 @@ type refCntBlockSet struct {
 	blocks map[util.Key]int
 }
 
-func NewRefCountBlockSet() set.BlockSet {
+func NewRefCountBlockSet() BlockSet {
 	return &refCntBlockSet{blocks: make(map[util.Key]int)}
 }
 

blocks/set/set.go

@@ -5,6 +5,8 @@ import (
 	"github.com/jbenet/go-ipfs/util"
 )
 
+var log = util.Logger("blockset")
+
 type BlockSet interface {
 	AddBlock(util.Key)
 	RemoveBlock(util.Key)

pin/pin.go

@@ -17,14 +17,19 @@ type pinner struct {
 	directPin  set.BlockSet
 	indirPin   set.BlockSet
 	dserv      *mdag.DAGService
+	dstore     ds.Datastore
 }
 
 func NewPinner(dstore ds.Datastore, serv *mdag.DAGService) Pinner {
+	rcset := set.NewDBWrapperSet(dstore, "/pinned/recurse/", set.NewSimpleBlockSet())
+	dirset := set.NewDBWrapperSet(dstore, "/pinned/direct/", set.NewSimpleBlockSet())
+	indset := set.NewDBWrapperSet(dstore, "/pinned/indirect/", set.NewRefCountBlockSet())
 	return &pinner{
-		recursePin: set.NewSimpleBlockSet(),
-		directPin:  set.NewSimpleBlockSet(),
-		indirPin:   NewRefCountBlockSet(),
+		recursePin: rcset,
+		directPin:  dirset,
+		indirPin:   indset,
 		dserv:      serv,
+		dstore:     dstore,
 	}
 }
 
@@ -52,7 +57,39 @@ func (p *pinner) Pin(node *mdag.Node, recurse bool) error {
 }
 
 func (p *pinner) Unpin(k util.Key, recurse bool) error {
-	panic("not yet implemented!")
+	if recurse {
+		p.recursePin.RemoveBlock(k)
+		node, err := p.dserv.Get(k)
+		if err != nil {
+			return err
+		}
+
+		return p.unpinLinks(node)
+	} else {
+		p.directPin.RemoveBlock(k)
+	}
+	return nil
+}
+
+func (p *pinner) unpinLinks(node *mdag.Node) error {
+	for _, l := range node.Links {
+		node, err := l.GetNode(p.dserv)
+		if err != nil {
+			return err
+		}
+
+		k, err := node.Key()
+		if err != nil {
+			return err
+		}
+
+		p.recursePin.RemoveBlock(k)
+
+		err = p.unpinLinks(node)
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }