reduce overhead casting between ds.Keys and strings

providers/providers.go

@@ -77,8 +77,8 @@ func NewProviderManager(ctx context.Context, local peer.ID, dstore ds.Batching)
 
 const providersKeyPrefix = "/providers/"
 
-func mkProvKey(k *cid.Cid) ds.Key {
-	return ds.NewKey(providersKeyPrefix + base32.RawStdEncoding.EncodeToString(k.Bytes()))
+func mkProvKey(k *cid.Cid) string {
+	return providersKeyPrefix + base32.RawStdEncoding.EncodeToString(k.Bytes())
 }
 
 func (pm *ProviderManager) Process() goprocess.Process {
@@ -112,13 +112,14 @@ func (pm *ProviderManager) getProvSet(k *cid.Cid) (*providerSet, error) {
 }
 
 func loadProvSet(dstore ds.Datastore, k *cid.Cid) (*providerSet, error) {
-	res, err := dstore.Query(dsq.Query{Prefix: mkProvKey(k).String()})
+	res, err := dstore.Query(dsq.Query{Prefix: mkProvKey(k)})
 	if err != nil {
 		return nil, err
 	}
 
 	out := newProviderSet()
 	for e := range res.Next() {
+
 		if e.Error != nil {
 			log.Error("got an error: ", e.Error)
 			continue
@@ -174,12 +175,12 @@ func (pm *ProviderManager) addProv(k *cid.Cid, p peer.ID) error {
 }
 
 func writeProviderEntry(dstore ds.Datastore, k *cid.Cid, p peer.ID, t time.Time) error {
-	dsk := mkProvKey(k).ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
+	dsk := mkProvKey(k) + "/" + base32.RawStdEncoding.EncodeToString([]byte(p))
 
 	buf := make([]byte, 16)
 	n := binary.PutVarint(buf, t.UnixNano())
 
-	return dstore.Put(dsk, buf[:n])
+	return dstore.Put(ds.RawKey(dsk), buf[:n])
 }
 
 func (pm *ProviderManager) deleteProvSet(k *cid.Cid) error {
@@ -187,7 +188,7 @@ func (pm *ProviderManager) deleteProvSet(k *cid.Cid) error {
 
 	res, err := pm.dstore.Query(dsq.Query{
 		KeysOnly: true,
-		Prefix:   mkProvKey(k).String(),
+		Prefix:   mkProvKey(k),
 	})
 
 	entries, err := res.Rest()
@@ -196,7 +197,7 @@ func (pm *ProviderManager) deleteProvSet(k *cid.Cid) error {
 	}
 
 	for _, e := range entries {
-		err := pm.dstore.Delete(ds.NewKey(e.Key))
+		err := pm.dstore.Delete(ds.RawKey(e.Key))
 		if err != nil {
 			log.Error("deleting provider set: ", err)
 		}

providers/providers_test.go

@@ -148,3 +148,54 @@ func TestProvidesExpire(t *testing.T) {
 		t.Fatal("expected everything to be cleaned out of the datastore")
 	}
 }
+
+/* This can be used for profiling. Keeping it commented out for now to avoid incurring extra CI time
+func TestLargeProvidersSet(t *testing.T) {
+	old := lruCacheSize
+	lruCacheSize = 10
+	defer func() { lruCacheSize = old }()
+
+	dirn, err := ioutil.TempDir("", "provtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	opts := &lds.Options{
+		NoSync:      true,
+		Compression: 1,
+	}
+	lds, err := lds.NewDatastore(dirn, opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = lds
+
+	defer func() {
+		os.RemoveAll(dirn)
+	}()
+
+	ctx := context.Background()
+	var peers []peer.ID
+	for i := 0; i < 3000; i++ {
+		peers = append(peers, peer.ID(fmt.Sprint(i)))
+	}
+
+	mid := peer.ID("myself")
+	p := NewProviderManager(ctx, mid, lds)
+	defer p.proc.Close()
+
+	var cids []*cid.Cid
+	for i := 0; i < 1000; i++ {
+		c := cid.NewCidV0(u.Hash([]byte(fmt.Sprint(i))))
+		cids = append(cids, c)
+		for _, pid := range peers {
+			p.AddProvider(ctx, c, pid)
+		}
+	}
+
+	for _, c := range cids {
+		_ = p.GetProviders(ctx, c)
+	}
+
+}
+*/