benchmarks_test.go 11 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
package bitswap

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"math/rand"
	"sync"
	"testing"
	"time"

12
	"github.com/ipfs/go-bitswap/testutil"
13

14
	bssession "github.com/ipfs/go-bitswap/session"
15
	tn "github.com/ipfs/go-bitswap/testnet"
16 17 18 19 20 21 22
	"github.com/ipfs/go-block-format"
	cid "github.com/ipfs/go-cid"
	blocksutil "github.com/ipfs/go-ipfs-blocksutil"
	delay "github.com/ipfs/go-ipfs-delay"
	mockrouting "github.com/ipfs/go-ipfs-routing/mock"
)

23
type fetchFunc func(b *testing.B, bs *Bitswap, ks []cid.Cid)
24

25
type distFunc func(b *testing.B, provs []Instance, blocks []blocks.Block)
26 27 28 29 30 31 32 33 34 35 36

type runStats struct {
	Dups    uint64
	MsgSent uint64
	MsgRecd uint64
	Time    time.Duration
	Name    string
}

var benchmarkLog []runStats

37
func BenchmarkDups2Nodes(b *testing.B) {
38
	benchmarkLog = nil
39
	fixedDelay := delay.Fixed(10 * time.Millisecond)
40
	b.Run("AllToAll-OneAtATime", func(b *testing.B) {
41
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, allToAll, oneAtATime)
42
	})
43
	b.Run("AllToAll-BigBatch", func(b *testing.B) {
44
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, allToAll, batchFetchAll)
45 46
	})

47
	b.Run("Overlap1-OneAtATime", func(b *testing.B) {
48
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, overlap1, oneAtATime)
49 50
	})

51
	b.Run("Overlap2-BatchBy10", func(b *testing.B) {
52
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, overlap2, batchFetchBy10)
53 54
	})

55
	b.Run("Overlap3-OneAtATime", func(b *testing.B) {
56
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, overlap3, oneAtATime)
57
	})
58
	b.Run("Overlap3-BatchBy10", func(b *testing.B) {
59
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, overlap3, batchFetchBy10)
60
	})
61
	b.Run("Overlap3-AllConcurrent", func(b *testing.B) {
62
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, overlap3, fetchAllConcurrent)
63
	})
64
	b.Run("Overlap3-BigBatch", func(b *testing.B) {
65
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, overlap3, batchFetchAll)
66
	})
67
	b.Run("Overlap3-UnixfsFetch", func(b *testing.B) {
68
		subtestDistributeAndFetch(b, 3, 100, fixedDelay, overlap3, unixfsFileFetch)
69
	})
70
	b.Run("10Nodes-AllToAll-OneAtATime", func(b *testing.B) {
71
		subtestDistributeAndFetch(b, 10, 100, fixedDelay, allToAll, oneAtATime)
72
	})
73
	b.Run("10Nodes-AllToAll-BatchFetchBy10", func(b *testing.B) {
74
		subtestDistributeAndFetch(b, 10, 100, fixedDelay, allToAll, batchFetchBy10)
75
	})
76
	b.Run("10Nodes-AllToAll-BigBatch", func(b *testing.B) {
77
		subtestDistributeAndFetch(b, 10, 100, fixedDelay, allToAll, batchFetchAll)
78
	})
79
	b.Run("10Nodes-AllToAll-AllConcurrent", func(b *testing.B) {
80
		subtestDistributeAndFetch(b, 10, 100, fixedDelay, allToAll, fetchAllConcurrent)
81
	})
82
	b.Run("10Nodes-AllToAll-UnixfsFetch", func(b *testing.B) {
83
		subtestDistributeAndFetch(b, 10, 100, fixedDelay, allToAll, unixfsFileFetch)
84
	})
85
	b.Run("10Nodes-OnePeerPerBlock-OneAtATime", func(b *testing.B) {
86
		subtestDistributeAndFetch(b, 10, 100, fixedDelay, onePeerPerBlock, oneAtATime)
87
	})
88
	b.Run("10Nodes-OnePeerPerBlock-BigBatch", func(b *testing.B) {
89
		subtestDistributeAndFetch(b, 10, 100, fixedDelay, onePeerPerBlock, batchFetchAll)
90
	})
91
	b.Run("10Nodes-OnePeerPerBlock-UnixfsFetch", func(b *testing.B) {
92
		subtestDistributeAndFetch(b, 10, 100, fixedDelay, onePeerPerBlock, unixfsFileFetch)
93
	})
94
	b.Run("200Nodes-AllToAll-BigBatch", func(b *testing.B) {
95
		subtestDistributeAndFetch(b, 200, 20, fixedDelay, allToAll, batchFetchAll)
96 97
	})
	out, _ := json.MarshalIndent(benchmarkLog, "", "  ")
98
	ioutil.WriteFile("tmp/benchmark.json", out, 0666)
99 100
}

101 102 103 104 105
const fastSpeed = 60 * time.Millisecond
const mediumSpeed = 200 * time.Millisecond
const slowSpeed = 800 * time.Millisecond
const superSlowSpeed = 4000 * time.Millisecond
const distribution = 20 * time.Millisecond
106 107 108 109 110 111 112
const fastBandwidth = 1250000.0
const fastBandwidthDeviation = 300000.0
const mediumBandwidth = 500000.0
const mediumBandwidthDeviation = 80000.0
const slowBandwidth = 100000.0
const slowBandwidthDeviation = 16500.0
const stdBlockSize = 8000
113 114

func BenchmarkDupsManyNodesRealWorldNetwork(b *testing.B) {
115
	benchmarkLog = nil
116 117 118 119
	fastNetworkDelayGenerator := tn.InternetLatencyDelayGenerator(
		mediumSpeed-fastSpeed, slowSpeed-fastSpeed,
		0.0, 0.0, distribution, nil)
	fastNetworkDelay := delay.Delay(fastSpeed, fastNetworkDelayGenerator)
120
	fastBandwidthGenerator := tn.VariableRateLimitGenerator(fastBandwidth, fastBandwidthDeviation, nil)
121 122 123 124
	averageNetworkDelayGenerator := tn.InternetLatencyDelayGenerator(
		mediumSpeed-fastSpeed, slowSpeed-fastSpeed,
		0.3, 0.3, distribution, nil)
	averageNetworkDelay := delay.Delay(fastSpeed, averageNetworkDelayGenerator)
125
	averageBandwidthGenerator := tn.VariableRateLimitGenerator(mediumBandwidth, mediumBandwidthDeviation, nil)
126 127 128 129
	slowNetworkDelayGenerator := tn.InternetLatencyDelayGenerator(
		mediumSpeed-fastSpeed, superSlowSpeed-fastSpeed,
		0.3, 0.3, distribution, nil)
	slowNetworkDelay := delay.Delay(fastSpeed, slowNetworkDelayGenerator)
130
	slowBandwidthGenerator := tn.VariableRateLimitGenerator(slowBandwidth, slowBandwidthDeviation, nil)
131 132

	b.Run("200Nodes-AllToAll-BigBatch-FastNetwork", func(b *testing.B) {
133
		subtestDistributeAndFetchRateLimited(b, 300, 200, fastNetworkDelay, fastBandwidthGenerator, stdBlockSize, allToAll, batchFetchAll)
134 135
	})
	b.Run("200Nodes-AllToAll-BigBatch-AverageVariableSpeedNetwork", func(b *testing.B) {
136
		subtestDistributeAndFetchRateLimited(b, 300, 200, averageNetworkDelay, averageBandwidthGenerator, stdBlockSize, allToAll, batchFetchAll)
137 138
	})
	b.Run("200Nodes-AllToAll-BigBatch-SlowVariableSpeedNetwork", func(b *testing.B) {
139
		subtestDistributeAndFetchRateLimited(b, 300, 200, slowNetworkDelay, slowBandwidthGenerator, stdBlockSize, allToAll, batchFetchAll)
140
	})
141 142
	out, _ := json.MarshalIndent(benchmarkLog, "", "  ")
	ioutil.WriteFile("tmp/rw-benchmark.json", out, 0666)
143 144 145
}

func subtestDistributeAndFetch(b *testing.B, numnodes, numblks int, d delay.D, df distFunc, ff fetchFunc) {
146
	start := time.Now()
147
	net := tn.VirtualNetwork(mockrouting.NewServer(), d)
148

149 150 151 152 153 154 155
	sg := NewTestSessionGenerator(net)
	defer sg.Close()

	bg := blocksutil.NewBlockGenerator()

	instances := sg.Instances(numnodes)
	blocks := bg.Blocks(numblks)
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
	runDistribution(b, instances, blocks, df, ff, start)
}

func subtestDistributeAndFetchRateLimited(b *testing.B, numnodes, numblks int, d delay.D, rateLimitGenerator tn.RateLimitGenerator, blockSize int64, df distFunc, ff fetchFunc) {
	start := time.Now()
	net := tn.RateLimitedVirtualNetwork(mockrouting.NewServer(), d, rateLimitGenerator)

	sg := NewTestSessionGenerator(net)
	defer sg.Close()

	instances := sg.Instances(numnodes)
	blocks := testutil.GenerateBlocksOfSize(numblks, blockSize)

	runDistribution(b, instances, blocks, df, ff, start)
}

func runDistribution(b *testing.B, instances []Instance, blocks []blocks.Block, df distFunc, ff fetchFunc, start time.Time) {

	numnodes := len(instances)
175 176 177

	fetcher := instances[numnodes-1]

178
	df(b, instances[:numnodes-1], blocks)
179 180 181 182 183 184

	var ks []cid.Cid
	for _, blk := range blocks {
		ks = append(ks, blk.Cid())
	}

185
	ff(b, fetcher.Exchange, ks)
186 187 188

	st, err := fetcher.Exchange.Stat()
	if err != nil {
189
		b.Fatal(err)
190 191 192 193 194 195 196 197
	}

	nst := fetcher.Exchange.network.Stats()
	stats := runStats{
		Time:    time.Now().Sub(start),
		MsgRecd: nst.MessagesRecvd,
		MsgSent: nst.MessagesSent,
		Dups:    st.DupBlksReceived,
198
		Name:    b.Name(),
199 200
	}
	benchmarkLog = append(benchmarkLog, stats)
201
	b.Logf("send/recv: %d / %d", nst.MessagesSent, nst.MessagesRecvd)
202
	if st.DupBlksReceived != 0 {
203
		b.Fatalf("got %d duplicate blocks!", st.DupBlksReceived)
204 205 206
	}
}

207
func allToAll(b *testing.B, provs []Instance, blocks []blocks.Block) {
208 209
	for _, p := range provs {
		if err := p.Blockstore().PutMany(blocks); err != nil {
210
			b.Fatal(err)
211 212 213 214 215 216
		}
	}
}

// overlap1 gives the first 75 blocks to the first peer, and the last 75 blocks
// to the second peer. This means both peers have the middle 50 blocks
217
func overlap1(b *testing.B, provs []Instance, blks []blocks.Block) {
218
	if len(provs) != 2 {
219
		b.Fatal("overlap1 only works with 2 provs")
220 221 222 223 224
	}
	bill := provs[0]
	jeff := provs[1]

	if err := bill.Blockstore().PutMany(blks[:75]); err != nil {
225
		b.Fatal(err)
226 227
	}
	if err := jeff.Blockstore().PutMany(blks[25:]); err != nil {
228
		b.Fatal(err)
229 230 231 232 233
	}
}

// overlap2 gives every even numbered block to the first peer, odd numbered
// blocks to the second.  it also gives every third block to both peers
234
func overlap2(b *testing.B, provs []Instance, blks []blocks.Block) {
235
	if len(provs) != 2 {
236
		b.Fatal("overlap2 only works with 2 provs")
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
	}
	bill := provs[0]
	jeff := provs[1]

	bill.Blockstore().Put(blks[0])
	jeff.Blockstore().Put(blks[0])
	for i, blk := range blks {
		if i%3 == 0 {
			bill.Blockstore().Put(blk)
			jeff.Blockstore().Put(blk)
		} else if i%2 == 1 {
			bill.Blockstore().Put(blk)
		} else {
			jeff.Blockstore().Put(blk)
		}
	}
}

255
func overlap3(b *testing.B, provs []Instance, blks []blocks.Block) {
256
	if len(provs) != 2 {
257
		b.Fatal("overlap3 only works with 2 provs")
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
	}

	bill := provs[0]
	jeff := provs[1]

	bill.Blockstore().Put(blks[0])
	jeff.Blockstore().Put(blks[0])
	for i, blk := range blks {
		if i%3 == 0 {
			bill.Blockstore().Put(blk)
			jeff.Blockstore().Put(blk)
		} else if i%2 == 1 {
			bill.Blockstore().Put(blk)
		} else {
			jeff.Blockstore().Put(blk)
		}
	}
}

// onePeerPerBlock picks a random peer to hold each block
// with this layout, we shouldnt actually ever see any duplicate blocks
// but we're mostly just testing performance of the sync algorithm
280
func onePeerPerBlock(b *testing.B, provs []Instance, blks []blocks.Block) {
281 282 283 284 285
	for _, blk := range blks {
		provs[rand.Intn(len(provs))].Blockstore().Put(blk)
	}
}

286
func oneAtATime(b *testing.B, bs *Bitswap, ks []cid.Cid) {
287
	ses := bs.NewSession(context.Background()).(*bssession.Session)
288 289 290
	for _, c := range ks {
		_, err := ses.GetBlock(context.Background(), c)
		if err != nil {
291
			b.Fatal(err)
292 293
		}
	}
294
	b.Logf("Session fetch latency: %s", ses.GetAverageLatency())
295 296 297
}

// fetch data in batches, 10 at a time
298
func batchFetchBy10(b *testing.B, bs *Bitswap, ks []cid.Cid) {
299 300 301 302
	ses := bs.NewSession(context.Background())
	for i := 0; i < len(ks); i += 10 {
		out, err := ses.GetBlocks(context.Background(), ks[i:i+10])
		if err != nil {
303
			b.Fatal(err)
304 305 306 307 308 309 310
		}
		for range out {
		}
	}
}

// fetch each block at the same time concurrently
311
func fetchAllConcurrent(b *testing.B, bs *Bitswap, ks []cid.Cid) {
312 313 314 315 316 317 318 319 320
	ses := bs.NewSession(context.Background())

	var wg sync.WaitGroup
	for _, c := range ks {
		wg.Add(1)
		go func(c cid.Cid) {
			defer wg.Done()
			_, err := ses.GetBlock(context.Background(), c)
			if err != nil {
321
				b.Fatal(err)
322 323 324 325 326 327
			}
		}(c)
	}
	wg.Wait()
}

328
func batchFetchAll(b *testing.B, bs *Bitswap, ks []cid.Cid) {
329 330 331
	ses := bs.NewSession(context.Background())
	out, err := ses.GetBlocks(context.Background(), ks)
	if err != nil {
332
		b.Fatal(err)
333 334 335 336 337 338
	}
	for range out {
	}
}

// simulates the fetch pattern of trying to sync a unixfs file graph as fast as possible
339
func unixfsFileFetch(b *testing.B, bs *Bitswap, ks []cid.Cid) {
340 341 342
	ses := bs.NewSession(context.Background())
	_, err := ses.GetBlock(context.Background(), ks[0])
	if err != nil {
343
		b.Fatal(err)
344 345 346 347
	}

	out, err := ses.GetBlocks(context.Background(), ks[1:11])
	if err != nil {
348
		b.Fatal(err)
349 350 351 352 353 354
	}
	for range out {
	}

	out, err = ses.GetBlocks(context.Background(), ks[11:])
	if err != nil {
355
		b.Fatal(err)
356 357 358 359
	}
	for range out {
	}
}