Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
dms3
go-graphsync
Commits
70d3ec7a
Unverified
Commit
70d3ec7a
authored
Oct 11, 2020
by
Hannah Howard
Committed by
GitHub
Oct 11, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat(responsecache): prune blocks more intelligently (#101)
parent
9529ffb3
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
21 additions
and
3 deletions
+21
-3
benchmarks/benchmark_test.go
benchmarks/benchmark_test.go
+6
-0
requestmanager/asyncloader/responsecache/responsecache.go
requestmanager/asyncloader/responsecache/responsecache.go
+6
-3
requestmanager/asyncloader/responsecache/responsecache_test.go
...stmanager/asyncloader/responsecache/responsecache_test.go
+4
-0
requestmanager/asyncloader/unverifiedblockstore/unverifiedblockstore.go
.../asyncloader/unverifiedblockstore/unverifiedblockstore.go
+5
-0
No files found.
benchmarks/benchmark_test.go
View file @
70d3ec7a
...
@@ -52,9 +52,15 @@ func BenchmarkRoundtripSuccess(b *testing.B) {
...
@@ -52,9 +52,15 @@ func BenchmarkRoundtripSuccess(b *testing.B) {
b
.
Run
(
"test-20-10000"
,
func
(
b
*
testing
.
B
)
{
b
.
Run
(
"test-20-10000"
,
func
(
b
*
testing
.
B
)
{
subtestDistributeAndFetch
(
ctx
,
b
,
20
,
delay
.
Fixed
(
0
),
time
.
Duration
(
0
),
allFilesUniformSize
(
10000
,
defaultUnixfsChunkSize
,
defaultUnixfsLinksPerLevel
),
tdm
)
subtestDistributeAndFetch
(
ctx
,
b
,
20
,
delay
.
Fixed
(
0
),
time
.
Duration
(
0
),
allFilesUniformSize
(
10000
,
defaultUnixfsChunkSize
,
defaultUnixfsLinksPerLevel
),
tdm
)
})
})
b
.
Run
(
"test-20-128MB"
,
func
(
b
*
testing
.
B
)
{
subtestDistributeAndFetch
(
ctx
,
b
,
10
,
delay
.
Fixed
(
0
),
time
.
Duration
(
0
),
allFilesUniformSize
(
128
*
(
1
<<
20
),
defaultUnixfsChunkSize
,
defaultUnixfsLinksPerLevel
),
tdm
)
})
b
.
Run
(
"test-p2p-stress-10-128MB"
,
func
(
b
*
testing
.
B
)
{
b
.
Run
(
"test-p2p-stress-10-128MB"
,
func
(
b
*
testing
.
B
)
{
p2pStrestTest
(
ctx
,
b
,
20
,
allFilesUniformSize
(
128
*
(
1
<<
20
),
1
<<
20
,
1024
),
tdm
)
p2pStrestTest
(
ctx
,
b
,
20
,
allFilesUniformSize
(
128
*
(
1
<<
20
),
1
<<
20
,
1024
),
tdm
)
})
})
b
.
Run
(
"test-p2p-stress-10-128MB-1KB-chunks"
,
func
(
b
*
testing
.
B
)
{
p2pStrestTest
(
ctx
,
b
,
10
,
allFilesUniformSize
(
128
*
(
1
<<
20
),
1
<<
10
,
1024
),
tdm
)
})
}
}
func
p2pStrestTest
(
ctx
context
.
Context
,
b
*
testing
.
B
,
numfiles
int
,
df
distFunc
,
tdm
*
tempDirMaker
)
{
func
p2pStrestTest
(
ctx
context
.
Context
,
b
*
testing
.
B
,
numfiles
int
,
df
distFunc
,
tdm
*
tempDirMaker
)
{
...
...
requestmanager/asyncloader/responsecache/responsecache.go
View file @
70d3ec7a
...
@@ -20,6 +20,7 @@ var log = logging.Logger("graphsync")
...
@@ -20,6 +20,7 @@ var log = logging.Logger("graphsync")
// as they come in and removing them as they are verified
// as they come in and removing them as they are verified
type
UnverifiedBlockStore
interface
{
type
UnverifiedBlockStore
interface
{
PruneBlocks
(
func
(
ipld
.
Link
)
bool
)
PruneBlocks
(
func
(
ipld
.
Link
)
bool
)
PruneBlock
(
ipld
.
Link
)
VerifyBlock
(
ipld
.
Link
)
([]
byte
,
error
)
VerifyBlock
(
ipld
.
Link
)
([]
byte
,
error
)
AddUnverifiedBlock
(
ipld
.
Link
,
[]
byte
)
AddUnverifiedBlock
(
ipld
.
Link
,
[]
byte
)
}
}
...
@@ -83,9 +84,11 @@ func (rc *ResponseCache) ProcessResponse(responses map[graphsync.RequestID]metad
...
@@ -83,9 +84,11 @@ func (rc *ResponseCache) ProcessResponse(responses map[graphsync.RequestID]metad
}
}
// prune unused blocks right away
// prune unused blocks right away
rc
.
unverifiedBlockStore
.
PruneBlocks
(
func
(
link
ipld
.
Link
)
bool
{
for
_
,
block
:=
range
blks
{
return
rc
.
linkTracker
.
BlockRefCount
(
link
)
==
0
if
rc
.
linkTracker
.
BlockRefCount
(
cidlink
.
Link
{
Cid
:
block
.
Cid
()})
==
0
{
})
rc
.
unverifiedBlockStore
.
PruneBlock
(
cidlink
.
Link
{
Cid
:
block
.
Cid
()})
}
}
rc
.
responseCacheLk
.
Unlock
()
rc
.
responseCacheLk
.
Unlock
()
}
}
requestmanager/asyncloader/responsecache/responsecache_test.go
View file @
70d3ec7a
...
@@ -31,6 +31,10 @@ func (ubs *fakeUnverifiedBlockStore) PruneBlocks(shouldPrune func(ipld.Link) boo
...
@@ -31,6 +31,10 @@ func (ubs *fakeUnverifiedBlockStore) PruneBlocks(shouldPrune func(ipld.Link) boo
}
}
}
}
func
(
ubs
*
fakeUnverifiedBlockStore
)
PruneBlock
(
link
ipld
.
Link
)
{
delete
(
ubs
.
inMemoryBlocks
,
link
)
}
func
(
ubs
*
fakeUnverifiedBlockStore
)
VerifyBlock
(
lnk
ipld
.
Link
)
([]
byte
,
error
)
{
func
(
ubs
*
fakeUnverifiedBlockStore
)
VerifyBlock
(
lnk
ipld
.
Link
)
([]
byte
,
error
)
{
data
,
ok
:=
ubs
.
inMemoryBlocks
[
lnk
]
data
,
ok
:=
ubs
.
inMemoryBlocks
[
lnk
]
if
!
ok
{
if
!
ok
{
...
...
requestmanager/asyncloader/unverifiedblockstore/unverifiedblockstore.go
View file @
70d3ec7a
...
@@ -38,6 +38,11 @@ func (ubs *UnverifiedBlockStore) PruneBlocks(shouldPrune func(ipld.Link) bool) {
...
@@ -38,6 +38,11 @@ func (ubs *UnverifiedBlockStore) PruneBlocks(shouldPrune func(ipld.Link) bool) {
}
}
}
}
// PruneBlock deletes an individual block from the store
func
(
ubs
*
UnverifiedBlockStore
)
PruneBlock
(
link
ipld
.
Link
)
{
delete
(
ubs
.
inMemoryBlocks
,
link
)
}
// VerifyBlock verifies the data for the given link as being part of a traversal,
// VerifyBlock verifies the data for the given link as being part of a traversal,
// removes it from the unverified store, and writes it to permaneant storage.
// removes it from the unverified store, and writes it to permaneant storage.
func
(
ubs
*
UnverifiedBlockStore
)
VerifyBlock
(
lnk
ipld
.
Link
)
([]
byte
,
error
)
{
func
(
ubs
*
UnverifiedBlockStore
)
VerifyBlock
(
lnk
ipld
.
Link
)
([]
byte
,
error
)
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment