Merge pull request #5526 from cboddy/feat/http_proxy_over_p2p

[http_proxy_over_p2p]

cmd/ipfs/daemon.go

@@ -570,6 +570,10 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
 		corehttp.CommandsROOption(*cctx),
 	}
 
+	if cfg.Experimental.P2pHttpProxy {
+		opts = append(opts, corehttp.ProxyOption())
+	}
+
 	if len(cfg.Gateway.RootRedirect) > 0 {
 		opts = append(opts, corehttp.RedirectOption("", cfg.Gateway.RootRedirect))
 	}

core/corehttp/proxy.go

+package corehttp
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+
+	core "github.com/ipfs/go-ipfs/core"
+
+	protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
+	p2phttp "gx/ipfs/QmcLYfmHLsaVRKGMZQovwEYhHAjWtRjg1Lij3pnzw5UkRD/go-libp2p-http"
+)
+
+// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer
+func ProxyOption() ServeOption {
+	return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
+		mux.HandleFunc("/p2p/", func(w http.ResponseWriter, request *http.Request) {
+			// parse request
+			parsedRequest, err := parseRequest(request)
+			if err != nil {
+				handleError(w, "failed to parse request", err, 400)
+				return
+			}
+
+			request.Host = "" // Let URL's Host take precedence.
+			request.URL.Path = parsedRequest.httpPath
+			target, err := url.Parse(fmt.Sprintf("libp2p://%s", parsedRequest.target))
+			if err != nil {
+				handleError(w, "failed to parse url", err, 400)
+				return
+			}
+
+			rt := p2phttp.NewTransport(ipfsNode.PeerHost, p2phttp.ProtocolOption(parsedRequest.name))
+			proxy := httputil.NewSingleHostReverseProxy(target)
+			proxy.Transport = rt
+			proxy.ServeHTTP(w, request)
+		})
+		return mux, nil
+	}
+}
+
+type proxyRequest struct {
+	target   string
+	name     protocol.ID
+	httpPath string // path to send to the proxy-host
+}
+
+// from the url path parse the peer-ID, name and http path
+// /p2p/$peer_id/http/$http_path
+// or
+// /p2p/$peer_id/x/$protocol/http/$http_path
+func parseRequest(request *http.Request) (*proxyRequest, error) {
+	path := request.URL.Path
+
+	split := strings.SplitN(path, "/", 5)
+	if len(split) < 5 {
+		return nil, fmt.Errorf("Invalid request path '%s'", path)
+	}
+
+	if split[3] == "http" {
+		return &proxyRequest{split[2], protocol.ID("/http"), split[4]}, nil
+	}
+
+	split = strings.SplitN(path, "/", 7)
+	if split[3] != "x" || split[5] != "http" {
+		return nil, fmt.Errorf("Invalid request path '%s'", path)
+	}
+
+	return &proxyRequest{split[2], protocol.ID("/x/" + split[4] + "/http"), split[6]}, nil
+}
+
+func handleError(w http.ResponseWriter, msg string, err error, code int) {
+	w.WriteHeader(code)
+	fmt.Fprintf(w, "%s: %s\n", msg, err)
+	log.Warningf("http proxy error: %s: %s", err)
+}

core/corehttp/proxy_test.go

+package corehttp
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/ipfs/go-ipfs/thirdparty/assert"
+
+	protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
+)
+
+type TestCase struct {
+	urlprefix string
+	target    string
+	name      string
+	path      string
+}
+
+var validtestCases = []TestCase{
+	{"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/http", "path/to/index.txt"},
+	{"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "path/to/index.txt"},
+	{"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "http/path/to/index.txt"},
+}
+
+func TestParseRequest(t *testing.T) {
+	for _, tc := range validtestCases {
+		url := tc.urlprefix + "/p2p/" + tc.target + tc.name + "/" + tc.path
+		req, _ := http.NewRequest("GET", url, strings.NewReader(""))
+
+		parsed, err := parseRequest(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.True(parsed.httpPath == tc.path, t, "proxy request path")
+		assert.True(parsed.name == protocol.ID(tc.name), t, "proxy request name")
+		assert.True(parsed.target == tc.target, t, "proxy request peer-id")
+	}
+}
+
+var invalidtestCases = []string{
+	"http://localhost:5001/p2p/http/foobar",
+	"http://localhost:5001/p2p/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/x/custom/foobar",
+}
+
+func TestParseRequestInvalidPath(t *testing.T) {
+	for _, tc := range invalidtestCases {
+		url := tc
+		req, _ := http.NewRequest("GET", url, strings.NewReader(""))
+
+		_, err := parseRequest(req)
+		if err == nil {
+			t.Fail()
+		}
+	}
+}

docs/experimental-features.md

@@ -21,6 +21,7 @@ the above issue.
 - [BadgerDB datastore](#badger-datastore)
 - [Private Networks](#private-networks)
 - [ipfs p2p](#ipfs-p2p)
+- [p2p http proxy](#p2p-http-proxy)
 - [Circuit Relay](#circuit-relay)
 - [Plugins](#plugins)
 - [Directory Sharding / HAMT](#directory-sharding-hamt)
@@ -382,6 +383,87 @@ with `ssh [user]@127.0.0.1 -p 2222`.
 
 ---
 
+## p2p http proxy
+
+Allows proxying of HTTP requests over p2p streams. This allows serving any standard http app over p2p streams.
+
+### State
+
+Experimental
+
+### In Version
+
+master, 0.4.19
+
+### How to enable
+
+The `p2p` command needs to be enabled in config:
+
+```sh
+> ipfs config --json Experimental.Libp2pStreamMounting true
+```
+
+On the client, the p2p http proxy needs to be enabled in the config:
+
+```sh
+> ipfs config --json Experimental.P2pHttpProxy true
+```
+
+### How to use
+
+**Netcat example:**
+
+First, pick a protocol name for your application. Think of the protocol name as
+a port number, just significantly more user-friendly. In this example, we're
+going to use `/http`.
+
+***Setup:***
+
+1. A "server" node with peer ID `$SERVER_ID`
+2. A "client" node.
+
+***On the "server" node:***
+
+First, start your application and have it listen for TCP connections on
+port `$APP_PORT`.
+
+Then, configure the p2p listener by running:
+
+```sh
+> ipfs p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$APP_PORT
+```
+
+This will configure IPFS to forward all incoming `/http` streams to
+`127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per incoming stream.
+
+***On the "client" node:***
+
+Next, have your application make a http request to `127.0.0.1:8080/p2p/$SERVER_ID/http/$FORWARDED_PATH`. This
+connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on
+the remote machine (which needs to be a http server!) with path `$FORWARDED_PATH`. You can test it with netcat:
+
+***On "server" node:***
+```sh
+> echo -e "HTTP/1.1 200\nContent-length: 11\n\nIPFS rocks!" | nc -l -p $APP_PORT
+```
+
+***On "client" node:***
+```sh
+> curl http://localhost:8080/p2p/$SERVER_ID/http/
+```
+
+You should now see the resulting http response: IPFS rocks!
+
+### Custom protocol names
+We also support use of protocol names of the form /x/$NAME/http where $NAME doesn't contain any "/"'s
+
+### Road to being a real feature
+- [ ] Needs p2p streams to graduate from experiments
+- [ ] Needs more people to use and report on how well it works / fits use cases
+- [ ] More documentation
+
+---
+
 ## Circuit Relay
 
 Allows peers to connect through an intermediate relay node when there

package.json

@@ -592,6 +592,12 @@
       "hash": "QmTqLBwme9BusYWdACqL62NFb8WV2Q72gXLsQVfC7vmCr4",
       "name": "iptb-plugins",
       "version": "1.0.5"
+    },
+    {
+      "author": "hsanjuan",
+      "hash": "QmcLYfmHLsaVRKGMZQovwEYhHAjWtRjg1Lij3pnzw5UkRD",
+      "name": "go-libp2p-http",
+      "version": "1.1.8"
     }
   ],
   "gxVersion": "0.10.0",

test/sharness/t0184-http-proxy-over-p2p.sh

+#!/usr/bin/env bash
+
+test_description="Test http proxy over p2p"
+
+. lib/test-lib.sh
+WEB_SERVE_PORT=5099
+IPFS_GATEWAY_PORT=5199
+SENDER_GATEWAY="http://127.0.0.1:$IPFS_GATEWAY_PORT"
+
+function show_logs() {
+
+    echo "*****************"
+    echo "  RECEIVER LOG  "
+    echo "*****************"
+    iptb logs 1
+    echo "*****************"
+    echo "  SENDER LOG  "
+    echo "*****************"
+    iptb logs 0
+    echo "*****************"
+    echo "REMOTE_SERVER LOG"
+    echo $REMOTE_SERVER_LOG
+    echo "*****************"
+    cat $REMOTE_SERVER_LOG
+}
+
+function start_http_server() {
+    REMOTE_SERVER_LOG="server.log"
+    rm -f $REMOTE_SERVER_LOG server_stdin
+
+    mkfifo server_stdin
+    nc -k -l 127.0.0.1 $WEB_SERVE_PORT 2>&1 > $REMOTE_SERVER_LOG < server_stdin &
+    REMOTE_SERVER_PID=$!
+    exec 7>server_stdin
+    rm server_stdin
+
+    while ! nc -z 127.0.0.1 $WEB_SERVE_PORT; do
+        go-sleep 100ms
+    done
+}
+
+function teardown_remote_server() {
+    exec 7<&-
+    kill $REMOTE_SERVER_PID > /dev/null 2>&1
+    wait $REMOTE_SERVER_PID || true
+}
+
+function serve_content() {
+    local body=$1
+    local status_code=${2:-"200 OK"}
+    local length=$((1 + ${#body}))
+    echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" >&7
+}
+
+function curl_check_response_code() {
+    local expected_status_code=$1
+    local path_stub=${2:-p2p/$RECEIVER_ID/http/index.txt}
+    local status_code=$(curl -s --write-out %{http_code} --output /dev/null $SENDER_GATEWAY/$path_stub)
+
+    if [[ "$status_code" -ne "$expected_status_code" ]];
+    then
+        echo "Found status-code "$status_code", expected "$expected_status_code
+        return 1
+    fi
+
+    return 0
+}
+
+function curl_send_proxy_request_and_check_response() {
+    local expected_status_code=$1
+    local expected_content=$2
+
+    #
+    # make a request to SENDER_IPFS via the proxy endpoint
+    #
+    CONTENT_PATH="retrieved-file"
+    STATUS_CODE="$(curl -s -o $CONTENT_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)"
+
+    #
+    # check status code
+    #
+    if [[ "$STATUS_CODE" -ne "$expected_status_code" ]];
+    then
+        echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code
+        return 1
+    fi
+
+    #
+    # check content
+    #
+    RESPONSE_CONTENT="$(tail -n 1 $CONTENT_PATH)"
+    if [[ "$RESPONSE_CONTENT" == "$expected_content" ]];
+    then
+        return 0
+    else
+        echo -e "Found response content:\n'"$RESPONSE_CONTENT"'\nthat differs from expected content:\n'"$expected_content"'"
+        return 1
+    fi
+}
+
+function curl_send_multipart_form_request() {
+    local expected_status_code=$1
+    local FILE_PATH="uploaded-file"
+    FILE_CONTENT="curl will send a multipart-form POST request when sending a file which is handy"
+    echo $FILE_CONTENT > $FILE_PATH
+    #
+    # send multipart form request
+    #
+    STATUS_CODE="$(curl -o /dev/null -s -F file=@$FILE_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)"
+    #
+    # check status code
+    #
+    if [[ "$STATUS_CODE" -ne "$expected_status_code" ]];
+    then
+        echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code
+        return 1
+    fi
+    #
+    # check request method
+    #
+    if ! grep "POST /index.txt" $REMOTE_SERVER_LOG > /dev/null;
+    then
+        echo "Remote server request method/resource path was incorrect"
+        show_logs
+        return 1
+    fi
+    #
+    # check request is multipart-form
+    #
+    if ! grep "Content-Type: multipart/form-data;" $REMOTE_SERVER_LOG > /dev/null;
+    then
+        echo "Request content-type was not multipart/form-data"
+        show_logs
+        return 1
+    fi
+    return 0
+}
+
+test_expect_success 'configure nodes' '
+    iptb testbed create -type localipfs -count 2 -force -init &&
+    ipfsi 0 config --json Experimental.Libp2pStreamMounting true &&
+    ipfsi 1 config --json Experimental.Libp2pStreamMounting true &&
+    ipfsi 0 config --json Experimental.P2pHttpProxy true
+    ipfsi 0 config --json Addresses.Gateway "[\"/ip4/127.0.0.1/tcp/$IPFS_GATEWAY_PORT\"]"
+'
+
+test_expect_success 'start and connect nodes' '
+    iptb start -wait && iptb connect 0 1
+'
+
+test_expect_success 'setup p2p listener on the receiver' '
+    ipfsi 1 p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT &&
+    ipfsi 1 p2p listen /x/custom/http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT
+'
+
+test_expect_success 'setup environment' '
+    RECEIVER_ID="$(iptb attr get 1 id)"
+'
+
+test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' '
+    curl_send_proxy_request_and_check_response 502 ""
+'
+
+test_expect_success 'start http server' '
+    start_http_server
+'
+
+test_expect_success 'handle proxy http request propogates error response from remote' '
+    serve_content "SORRY GUYS, I LOST IT" "404 Not Found" &&
+    curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT"
+'
+
+test_expect_success 'handle proxy http request ' '
+    serve_content "THE WOODS ARE LOVELY DARK AND DEEP" &&
+    curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP"
+'
+
+test_expect_success 'handle proxy http request invalid request' '
+    curl_check_response_code 400 p2p/DERPDERPDERP
+'
+
+test_expect_success 'handle proxy http request unknown proxy peer ' '
+    curl_check_response_code 502 p2p/unknown_peer/http/index.txt
+'
+
+test_expect_success 'handle proxy http request to custom protocol' '
+    serve_content "THE WOODS ARE LOVELY DARK AND DEEP" &&
+    curl_check_response_code 200 p2p/$RECEIVER_ID/x/custom/http/index.txt
+'
+
+test_expect_success 'handle proxy http request to missing protocol' '
+    serve_content "THE WOODS ARE LOVELY DARK AND DEEP" &&
+    curl_check_response_code 502 p2p/$RECEIVER_ID/x/missing/http/index.txt
+'
+
+test_expect_success 'handle proxy http request missing the /http' '
+    curl_check_response_code 400 p2p/$RECEIVER_ID/x/custom/index.txt
+'
+
+test_expect_success 'handle multipart/form-data http request' '
+    serve_content "OK" &&
+    curl_send_multipart_form_request 200
+'
+
+test_expect_success 'stop http server' '
+    teardown_remote_server
+'
+
+test_expect_success 'stop nodes' '
+    iptb stop
+'
+
+test_done