Merge pull request #4320 from ipfs/fix/gateway-seeker

gateway: fix seeker can't seek on specific files

core/corehttp/gateway_handler.go

@@ -268,7 +268,7 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
 
 	if !dir {
 		name := gopath.Base(urlPath)
-		http.ServeContent(w, r, name, modtime, dr)
+		i.serveFile(w, r, name, modtime, dr)
 		return
 	}
 
@@ -372,6 +372,34 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
 	}
 }
 
+type sizeReadSeeker interface {
+	Size() uint64
+
+	io.ReadSeeker
+}
+
+type sizeSeeker struct {
+	sizeReadSeeker
+}
+
+func (s *sizeSeeker) Seek(offset int64, whence int) (int64, error) {
+	if whence == io.SeekEnd && offset == 0 {
+		return int64(s.Size()), nil
+	}
+
+	return s.sizeReadSeeker.Seek(offset, whence)
+}
+
+func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) {
+	if sp, ok := content.(sizeReadSeeker); ok {
+		content = &sizeSeeker{
+			sizeReadSeeker: sp,
+		}
+	}
+
+	http.ServeContent(w, req, name, modtime, content)
+}
+
 func (i *gatewayHandler) postHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	p, err := i.api.Unixfs().Add(ctx, r.Body)
 	if err != nil {

merkledag/merkledag.go

@@ -307,11 +307,14 @@ func GetNodes(ctx context.Context, ds DAGService, keys []*cid.Cid) []NodeGetter
 
 // Remove duplicates from a list of keys
 func dedupeKeys(cids []*cid.Cid) []*cid.Cid {
+	out := make([]*cid.Cid, 0, len(cids))
 	set := cid.NewSet()
 	for _, c := range cids {
-		set.Add(c)
+		if set.Visit(c) {
+			out = append(out, c)
+		}
 	}
-	return set.Keys()
+	return out
 }
 
 func newNodePromise(ctx context.Context) NodeGetter {

test/sharness/t0110-gateway-data/foo.block

+
+	foo
\ No newline at end of file

test/sharness/t0110-gateway.sh

@@ -124,7 +124,7 @@ test_expect_success "HEAD 'index.html' has no content" '
 # test ipfs readonly api
 
 test_curl_gateway_api() {
-    curl -sfo actual "http://127.0.0.1:$port/api/v0/$1"
+  curl -sfo actual "http://127.0.0.1:$port/api/v0/$1"
 }
 
 test_expect_success "get IPFS directory file through readonly API succeeds" '
@@ -140,7 +140,7 @@ test_expect_success "refs IPFS directory file through readonly API succeeds" '
 '
 
 test_expect_success "test gateway api is sanitized" '
-for cmd in "add" "block/put" "bootstrap" "config" "dht" "diag" "dns" "get" "id" "mount" "name/publish" "object/put" "object/new" "object/patch" "pin" "ping" "refs/local" "repo" "resolve" "stats" "swarm"  "file" "update" "version" "bitswap"; do
+  for cmd in "add" "block/put" "bootstrap" "config" "dht" "diag" "dns" "get" "id" "mount" "name/publish" "object/put" "object/new" "object/patch" "pin" "ping" "refs/local" "repo" "resolve" "stats" "swarm"  "file" "update" "version" "bitswap"; do
     test_curl_resp_http_code "http://127.0.0.1:$port/api/v0/$cmd" "HTTP/1.1 404 Not Found"
   done
 '
@@ -155,6 +155,17 @@ test_expect_success "try fetching it from gateway" '
   test_cmp rfile ffile
 '
 
+test_expect_success "Add compact blocks" '
+  ipfs block put ../t0110-gateway-data/foo.block &&
+  FOO2_HASH=$(ipfs block put ../t0110-gateway-data/foofoo.block) &&
+  printf "foofoo" > expected
+'
+
+test_expect_success "GET compact blocks succeeds" '
+  curl -o actual "http://127.0.0.1:$port/ipfs/$FOO2_HASH" &&
+  test_cmp expected actual
+'
+
 test_kill_ipfs_daemon
 
 test_done

unixfs/io/pbdagreader.go

@@ -188,6 +188,9 @@ func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) {
 		if offset < 0 {
 			return -1, errors.New("Invalid offset")
 		}
+		if offset == dr.offset {
+			return offset, nil
+		}
 
 		// Grab cached protobuf object (solely to make code look cleaner)
 		pb := dr.pbdata
@@ -239,11 +242,24 @@ func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) {
 		return offset, nil
 	case io.SeekCurrent:
 		// TODO: be smarter here
+		if offset == 0 {
+			return dr.offset, nil
+		}
+
 		noffset := dr.offset + offset
 		return dr.Seek(noffset, io.SeekStart)
 	case io.SeekEnd:
 		noffset := int64(dr.pbdata.GetFilesize()) - offset
-		return dr.Seek(noffset, io.SeekStart)
+		n, err := dr.Seek(noffset, io.SeekStart)
+
+		// Return negative number if we can't figure out the file size. Using io.EOF
+		// for this seems to be good(-enough) solution as it's only returned by
+		// precalcNextBuf when we step out of file range.
+		// This is needed for gateway to function properly
+		if err == io.EOF && *dr.pbdata.Type == ftpb.Data_File {
+			return -1, nil
+		}
+		return n, err
 	default:
 		return 0, errors.New("invalid whence")
 	}