Merge pull request #1621 from ipfs/real-trailers

use go's built in handling of trailers and dont do custom chunking

.travis.yml

@@ -7,8 +7,7 @@ os:
 language: go
 
 go:
-#  - 1.3
-  - 1.4
+    - 1.5.1
 
 env:
   - TEST_NO_FUSE=1 TEST_VERBOSE=1 TEST_SUITE=test_go_expensive

README.md

@@ -35,7 +35,7 @@ From there:
 
 ### Prerequisite: Install Go
 
-First, you'll need go. If you don't have it: [Download Go 1.4+](https://golang.org/dl/).
+First, you'll need go. If you don't have it: [Download Go 1.5.1+](https://golang.org/dl/).
 
 You'll need to add Go's bin directories to your `$PATH` environment variable e.g., by adding these lines to your `/etc/profile` (for a system-wide installation) or `$HOME/.profile`:
 ```

circle.yml

@@ -3,11 +3,17 @@ machine:
     TEST_NO_FUSE: 1
     TEST_VERBOSE: 1
     TRAVIS: 1
+  post:
+    - sudo rm -rf /usr/local/go
+    - if [ ! -e go1.5.linux-amd64.tar.gz ]; then curl -o go1.5.linux-amd64.tar.gz https://storage.googleapis.com/golang/go1.5.linux-amd64.tar.gz; fi
+    - sudo tar -C /usr/local -xzf go1.5.linux-amd64.tar.gz
 
 dependencies:
   pre:
     # setup ipv6
     - sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 net.ipv6.conf.all.disable_ipv6=0
+  cache_directories:
+    - ~/go1.5.linux-amd64.tar.gz
 
 test:
   override:

cmd/ipfs/go_req.go

+// +build !go1.5
+
+`IPFS needs to be built with go version 1.5 or greater`

commands/http/handler.go

 package http
 
 import (
-	"bufio"
 	"errors"
 	"fmt"
 	"io"
@@ -205,6 +204,10 @@ func sendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req
 	}
 
 	h := w.Header()
+
+	// Set up our potential trailer
+	h.Set("Trailer", StreamErrHeader)
+
 	if res.Length() > 0 {
 		h.Set(contentLengthHeader, strconv.FormatUint(res.Length(), 10))
 	}
@@ -237,82 +240,12 @@ func sendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req
 		return
 	}
 
-	if err := writeResponse(status, w, out); err != nil {
-		if strings.Contains(err.Error(), "broken pipe") {
-			log.Info("client disconnect while writing stream ", err)
-			return
-		}
-
-		log.Error("error while writing stream ", err)
-	}
-}
-
-// Copies from an io.Reader to a http.ResponseWriter.
-// Flushes chunks over HTTP stream as they are read (if supported by transport).
-func writeResponse(status int, w http.ResponseWriter, out io.Reader) error {
-	// hijack the connection so we can write our own chunked output and trailers
-	hijacker, ok := w.(http.Hijacker)
-	if !ok {
-		log.Error("Failed to create hijacker! cannot continue!")
-		return errors.New("Could not create hijacker")
-	}
-	conn, writer, err := hijacker.Hijack()
+	w.WriteHeader(status)
+	_, err = io.Copy(w, out)
 	if err != nil {
-		return err
-	}
-	defer conn.Close()
-
-	// write status
-	writer.WriteString(fmt.Sprintf("HTTP/1.1 %d %s\r\n", status, http.StatusText(status)))
-
-	// Write out headers
-	w.Header().Write(writer)
-
-	// end of headers
-	writer.WriteString("\r\n")
-
-	// write body
-	streamErr := writeChunks(out, writer)
-
-	// close body
-	writer.WriteString("0\r\n")
-
-	// if there was a stream error, write out an error trailer. hopefully
-	// the client will pick it up!
-	if streamErr != nil {
-		writer.WriteString(StreamErrHeader + ": " + sanitizedErrStr(streamErr) + "\r\n")
-	}
-	writer.WriteString("\r\n") // close response
-	writer.Flush()
-	return streamErr
-}
-
-func writeChunks(r io.Reader, w *bufio.ReadWriter) error {
-	buf := make([]byte, 32*1024)
-	for {
-		n, err := r.Read(buf)
-
-		if n > 0 {
-			length := fmt.Sprintf("%x\r\n", n)
-			w.WriteString(length)
-
-			_, err := w.Write(buf[0:n])
-			if err != nil {
-				return err
-			}
-
-			w.WriteString("\r\n")
-			w.Flush()
-		}
-
-		if err != nil && err != io.EOF {
-			return err
-		}
-		if err == io.EOF {
-			break
-		}
+		log.Error("err: ", err)
+		w.Header().Set(StreamErrHeader, sanitizedErrStr(err))
 	}
-	return nil
 }
 
 func sanitizedErrStr(err error) string {

test/sharness/t0230-channel-streaming-http-content-type.sh

@@ -22,11 +22,14 @@ test_ls_cmd() {
 	test_expect_success "Text encoded channel-streaming command output looks good" '
 		printf "HTTP/1.1 200 OK\r\n" >expected_output &&
 		printf "Content-Type: text/plain\r\n" >>expected_output &&
+		printf "Trailer: X-Stream-Error\r\n" >>expected_output &&
 		printf "Transfer-Encoding: chunked\r\n" >>expected_output &&
 		printf "X-Chunked-Output: 1\r\n" >>expected_output &&
+		printf "Transfer-Encoding: chunked\r\n" >>expected_output &&
 		printf "\r\n" >>expected_output &&
 		echo QmRmPLc1FsPAn8F8F9DQDEYADNX5ER2sgqiokEvqnYknVW >>expected_output &&
-		test_cmp expected_output actual_output
+		cat actual_output | grep -vE Date > cleaned_output &&
+		test_cmp expected_output cleaned_output
 	'
 
 	test_expect_success "JSON encoded channel-streaming command succeeds" '
@@ -39,8 +42,10 @@ test_ls_cmd() {
 	test_expect_success "JSON encoded channel-streaming command output looks good" '
 		printf "HTTP/1.1 200 OK\r\n" >expected_output &&
 		printf "Content-Type: application/json\r\n" >>expected_output &&
+		printf "Trailer: X-Stream-Error\r\n" >>expected_output &&
 		printf "Transfer-Encoding: chunked\r\n" >>expected_output &&
 		printf "X-Chunked-Output: 1\r\n" >>expected_output &&
+		printf "Transfer-Encoding: chunked\r\n" >>expected_output &&
 		printf "\r\n" >>expected_output &&
 		cat <<-\EOF >>expected_output &&
 			{
@@ -48,8 +53,10 @@ test_ls_cmd() {
 			  "Err": ""
 			}
 		EOF
+		printf "\n" >> expected_output &&
 		perl -pi -e '"'"'chomp if eof'"'"' expected_output &&
-		test_cmp expected_output actual_output
+		cat actual_output | grep -vE Date > cleaned_output &&
+		test_cmp expected_output cleaned_output
 	'
 }