Flatten multipart file transfers

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

cli/parse.go

@@ -44,7 +44,17 @@ func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *c
 		}
 	}
 
-	stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, root)
+	// if '--hidden' is provided, enumerate hidden paths
+	hiddenOpt := req.Option("hidden")
+	hidden := false
+	if hiddenOpt != nil {
+		hidden, _, err = hiddenOpt.Bool()
+		if err != nil {
+			return req, nil, nil, u.ErrCast()
+		}
+	}
+
+	stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, hidden, root)
 	if err != nil {
 		return req, cmd, path, err
 	}
@@ -223,7 +233,7 @@ func parseOpts(args []string, root *cmds.Command) (
 	return
 }
 
-func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool, root *cmds.Command) ([]string, []files.File, error) {
+func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) {
 	// ignore stdin on Windows
 	if runtime.GOOS == "windows" {
 		stdin = nil
@@ -308,7 +318,7 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
 				// treat stringArg values as file paths
 				fpath := inputs[0]
 				inputs = inputs[1:]
-				file, err := appendFile(fpath, argDef, recursive)
+				file, err := appendFile(fpath, argDef, recursive, hidden)
 				if err != nil {
 					return nil, nil, err
 				}
@@ -389,7 +399,7 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err
 const notRecursiveFmtStr = "'%s' is a directory, use the '-%s' flag to specify directories"
 const dirNotSupportedFmtStr = "Invalid path '%s', argument '%s' does not support directories"
 
-func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File, error) {
+func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (files.File, error) {
 	fpath = filepath.ToSlash(filepath.Clean(fpath))
 
 	if fpath == "." {
@@ -414,7 +424,7 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File
 		}
 	}
 
-	return files.NewSerialFile(path.Base(fpath), fpath, stat)
+	return files.NewSerialFile(path.Base(fpath), fpath, hidden, stat)
 }
 
 // isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),

files/multipartfile.go

 package files
 
 import (
+	"io"
 	"io/ioutil"
 	"mime"
 	"mime/multipart"
-	"net/http"
 	"net/url"
 )
 
@@ -12,7 +12,8 @@ const (
 	multipartFormdataType = "multipart/form-data"
 	multipartMixedType    = "multipart/mixed"
 
-	applicationSymlink = "application/symlink"
+	applicationDirectory = "application/x-directory"
+	applicationSymlink   = "application/symlink"
 
 	contentTypeHeader = "Content-Type"
 )
@@ -45,40 +46,33 @@ func NewFileFromPart(part *multipart.Part) (File, error) {
 		}, nil
 	}
 
-	var params map[string]string
 	var err error
-	f.Mediatype, params, err = mime.ParseMediaType(contentType)
+	f.Mediatype, _, err = mime.ParseMediaType(contentType)
 	if err != nil {
 		return nil, err
 	}
 
-	if f.IsDirectory() {
-		boundary, found := params["boundary"]
-		if !found {
-			return nil, http.ErrMissingBoundary
-		}
-
-		f.Reader = multipart.NewReader(part, boundary)
-	}
-
 	return f, nil
 }
 
 func (f *MultipartFile) IsDirectory() bool {
-	return f.Mediatype == multipartFormdataType || f.Mediatype == multipartMixedType
+	return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory
 }
 
 func (f *MultipartFile) NextFile() (File, error) {
 	if !f.IsDirectory() {
 		return nil, ErrNotDirectory
 	}
+	if f.Reader != nil {
+		part, err := f.Reader.NextPart()
+		if err != nil {
+			return nil, err
+		}
 
-	part, err := f.Reader.NextPart()
-	if err != nil {
-		return nil, err
+		return NewFileFromPart(part)
 	}
 
-	return NewFileFromPart(part)
+	return nil, io.EOF
 }
 
 func (f *MultipartFile) FileName() string {

files/serialfile.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 )
 
@@ -18,9 +19,10 @@ type serialFile struct {
 	files   []os.FileInfo
 	stat    os.FileInfo
 	current *File
+	hidden  bool
 }
 
-func NewSerialFile(name, path string, stat os.FileInfo) (File, error) {
+func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) {
 	switch mode := stat.Mode(); {
 	case mode.IsRegular():
 		file, err := os.Open(path)
@@ -35,7 +37,7 @@ func NewSerialFile(name, path string, stat os.FileInfo) (File, error) {
 		if err != nil {
 			return nil, err
 		}
-		return &serialFile{name, path, contents, stat, nil}, nil
+		return &serialFile{name, path, contents, stat, nil, hidden}, nil
 	case mode&os.ModeSymlink != 0:
 		target, err := os.Readlink(path)
 		if err != nil {
@@ -68,6 +70,15 @@ func (f *serialFile) NextFile() (File, error) {
 	stat := f.files[0]
 	f.files = f.files[1:]
 
+	for !f.hidden && strings.HasPrefix(stat.Name(), ".") {
+		if len(f.files) == 0 {
+			return nil, io.EOF
+		}
+
+		stat = f.files[0]
+		f.files = f.files[1:]
+	}
+
 	// open the next file
 	fileName := filepath.ToSlash(filepath.Join(f.name, stat.Name()))
 	filePath := filepath.ToSlash(filepath.Join(f.path, stat.Name()))
@@ -75,7 +86,7 @@ func (f *serialFile) NextFile() (File, error) {
 	// recursively call the constructor on the next file
 	// if it's a regular file, we will open it as a ReaderFile
 	// if it's a directory, files in it will be opened serially
-	sf, err := NewSerialFile(fileName, filePath, stat)
+	sf, err := NewSerialFile(fileName, filePath, f.hidden, stat)
 	if err != nil {
 		return nil, err
 	}
@@ -94,7 +105,7 @@ func (f *serialFile) FullPath() string {
 }
 
 func (f *serialFile) Read(p []byte) (int, error) {
-	return 0, ErrNotReader
+	return 0, io.EOF
 }
 
 func (f *serialFile) Close() error {

files/slicefile.go

@@ -41,7 +41,7 @@ func (f *SliceFile) FullPath() string {
 }
 
 func (f *SliceFile) Read(p []byte) (int, error) {
-	return 0, ErrNotReader
+	return 0, io.EOF
 }
 
 func (f *SliceFile) Close() error {

http/client.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
-	path "github.com/ipfs/go-ipfs/path"
 	config "github.com/ipfs/go-ipfs/repo/config"
 
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
@@ -86,8 +85,8 @@ func (c *client) Send(req cmds.Request) (cmds.Response, error) {
 		reader = fileReader
 	}
 
-	pth := path.Join(req.Path())
-	url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, pth, query)
+	path := strings.Join(req.Path(), "/")
+	url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, path, query)
 
 	httpReq, err := http.NewRequest("POST", url, reader)
 	if err != nil {

http/multifilereader.go

@@ -17,7 +17,7 @@ import (
 type MultiFileReader struct {
 	io.Reader
 
-	files       files.File
+	files       []files.File
 	currentFile io.Reader
 	buf         bytes.Buffer
 	mpWriter    *multipart.Writer
@@ -34,7 +34,7 @@ type MultiFileReader struct {
 // if `form` is false, the Content-Type will be 'multipart/mixed'.
 func NewMultiFileReader(file files.File, form bool) *MultiFileReader {
 	mfr := &MultiFileReader{
-		files: file,
+		files: []files.File{file},
 		form:  form,
 		mutex: &sync.Mutex{},
 	}
@@ -54,34 +54,41 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
 
 	// if the current file isn't set, advance to the next file
 	if mfr.currentFile == nil {
-		file, err := mfr.files.NextFile()
-		if err == io.EOF {
-			mfr.mpWriter.Close()
-			mfr.closed = true
-		} else if err != nil {
-			return 0, err
+		var file files.File
+		for file == nil {
+			if len(mfr.files) == 0 {
+				mfr.mpWriter.Close()
+				mfr.closed = true
+				return mfr.buf.Read(buf)
+			}
+
+			nextfile, err := mfr.files[len(mfr.files)-1].NextFile()
+			if err == io.EOF {
+				mfr.files = mfr.files[:len(mfr.files)-1]
+				continue
+			} else if err != nil {
+				return 0, err
+			}
+
+			file = nextfile
 		}
 
 		// handle starting a new file part
 		if !mfr.closed {
 
 			var contentType string
-			if s, ok := file.(*files.Symlink); ok {
-				mfr.currentFile = s
-
+			if _, ok := file.(*files.Symlink); ok {
 				contentType = "application/symlink"
 			} else if file.IsDirectory() {
-				// if file is a directory, create a multifilereader from it
-				// (using 'multipart/mixed')
-				nmfr := NewMultiFileReader(file, false)
-				mfr.currentFile = nmfr
-				contentType = fmt.Sprintf("multipart/mixed; boundary=%s", nmfr.Boundary())
+				mfr.files = append(mfr.files, file)
+				contentType = "application/x-directory"
 			} else {
 				// otherwise, use the file as a reader to read its contents
-				mfr.currentFile = file
 				contentType = "application/octet-stream"
 			}
 
+			mfr.currentFile = file
+
 			// write the boundary and headers
 			header := make(textproto.MIMEHeader)
 			filename := url.QueryEscape(file.FileName())