core/commands/unixfs: Add 'ipfs unixfs ls ...'

This is similar to 'ipfs ls ...', but it:

* Lists file sizes that match the content size:

    $ ipfs --encoding=json unixfs ls /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4
    {
      "Objects": [
        {
          "Argument": "/ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4",
          "Links": [
            {
              "Name": "busybox",
              "Hash": "QmPbjmmci73roXf9VijpyQGgRJZthiQfnEetaMRGoGYV5a",
              "Size": 1947624,
              "Type": 2
            }
          ]
        }
      ]
    }
    $ ipfs cat /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4/busybox | wc -c
    1947624

  'ipfs ls ...', on the other hand, is using the Merkle-descendant
  size, which also includes fanout links and the typing information
  unixfs objects store in their Data:

    $ ipfs --encoding=json ls /ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4
    {
      "Objects": [
        {
          "Hash": "/ipfs/QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4",
          "Links": [
            {
              "Name": "busybox",
              "Hash": "QmPbjmmci73roXf9VijpyQGgRJZthiQfnEetaMRGoGYV5a",
              "Size": 1948128,
              "Type": 2
            }
          ]
        }
      ]
    }

* Has a simpler text output corresponding to POSIX ls [1]:

    $ ipfs unixfs ls /ipfs/QmV2FrBtvue5ve7vxbAzKz3mTdWq8wfMNPwYd8d9KHksCF/gentoo/stage3/amd64/2015-04-02
    bin
    dev
    etc
    proc
    run
    sys
    $ ipfs ls /ipfs/QmV2FrBtvue5ve7vxbAzKz3mTdWq8wfMNPwYd8d9KHksCF/gentoo/stage3/amd64/2015-04-02
    QmSRCHG21Sbqm3EJG9aEBo4vS7Fqu86pAjqf99MyCdNxZ4 1948183 bin/
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4       dev/
    QmUz1Z5jnQEjwr78fiMk5babwjJBDmhN5sx5HvPiTGGGjM 1207    etc/
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4       proc/
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4       run/
    QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn 4       sys/

  The minimal output allows us to start off with POSIX compliance and
  then add options (which may or may not be POSIX compatible) to
  adjust the output format as we get a better feel for what we need
  ([2] through [3]).

[1]: http://pubs.opengroup.org/onlinepubs/9699919799/utilities/ls.html
[2]: https://botbot.me/freenode/ipfs/2015-06-12/?msg=41724727&page=5
[3]: https://botbot.me/freenode/ipfs/2015-06-12/?msg=41725146&page=5

License: MIT
Signed-off-by: W. Trevor King <wking@tremily.us>

core/commands/root.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
+	unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs"
 	evlog "github.com/ipfs/go-ipfs/thirdparty/eventlog"
 )
 
@@ -35,6 +36,7 @@ DATA STRUCTURE COMMANDS
 
     block         Interact with raw blocks in the datastore
     object        Interact with raw dag nodes
+    unixfs        Interact with Unix filesystem objects
 
 ADVANCED COMMANDS
 
@@ -102,6 +104,7 @@ var rootSubcommands = map[string]*cmds.Command{
 	"stats":     StatsCmd,
 	"swarm":     SwarmCmd,
 	"tour":      tourCmd,
+	"unixfs":    unixfs.UnixFSCmd,
 	"update":    UpdateCmd,
 	"version":   VersionCmd,
 	"bitswap":   BitswapCmd,

core/commands/unixfs/ls.go

+package unixfs
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"text/tabwriter"
+	"time"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+
+	cmds "github.com/ipfs/go-ipfs/commands"
+	core "github.com/ipfs/go-ipfs/core"
+	path "github.com/ipfs/go-ipfs/path"
+	unixfs "github.com/ipfs/go-ipfs/unixfs"
+	unixfspb "github.com/ipfs/go-ipfs/unixfs/pb"
+)
+
+type LsLink struct {
+	Name, Hash string
+	Size       uint64
+	Type       unixfspb.Data_DataType
+}
+
+type LsObject struct {
+	Argument string
+	Links    []LsLink
+}
+
+type LsOutput struct {
+	Objects []*LsObject
+}
+
+var LsCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "List directory contents for Unix-filesystem objects",
+		ShortDescription: `
+Retrieves the object named by <ipfs-or-ipns-path> and displays the
+contents with the following format:
+
+  <hash> <type> <size> <name>
+
+For files, the child size is the total size of the file contents.  For
+directories, the child size is the IPFS link size.
+`,
+	},
+
+	Arguments: []cmds.Argument{
+		cmds.StringArg("ipfs-path", true, true, "The path to the IPFS object(s) to list links from").EnableStdin(),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		node, err := req.Context().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		paths := req.Arguments()
+
+		output := make([]*LsObject, len(paths))
+		for i, fpath := range paths {
+			dagnode, err := core.Resolve(req.Context().Context, node, path.Path(fpath))
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
+			output[i] = &LsObject{
+				Argument: fpath,
+				Links:    make([]LsLink, len(dagnode.Links)),
+			}
+			for j, link := range dagnode.Links {
+				ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+				defer cancel()
+				link.Node, err = link.GetNode(ctx, node.DAG)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+				d, err := unixfs.FromBytes(link.Node.Data)
+				if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
+				}
+				lsLink := LsLink{
+					Name: link.Name,
+					Hash: link.Hash.B58String(),
+					Type: d.GetType(),
+				}
+				if lsLink.Type == unixfspb.Data_File {
+					lsLink.Size = d.GetFilesize()
+				} else {
+					lsLink.Size = link.Size
+				}
+				output[i].Links[j] = lsLink
+			}
+		}
+
+		res.SetOutput(&LsOutput{Objects: output})
+	},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+
+			output := res.Output().(*LsOutput)
+			buf := new(bytes.Buffer)
+			w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0)
+			for _, object := range output.Objects {
+				if len(output.Objects) > 1 {
+					fmt.Fprintf(w, "%s:\n", object.Argument)
+				}
+				for _, link := range object.Links {
+					fmt.Fprintf(w, "%s\n", link.Name)
+				}
+				if len(output.Objects) > 1 {
+					fmt.Fprintln(w)
+				}
+			}
+			w.Flush()
+
+			return buf, nil
+		},
+	},
+	Type: LsOutput{},
+}

core/commands/unixfs/unixfs.go

+package unixfs
+
+import cmds "github.com/ipfs/go-ipfs/commands"
+
+var UnixFSCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Interact with ipfs objects representing Unix filesystems",
+		ShortDescription: `
+'ipfs unixfs' provides a familar interface to filesystems represtented
+by IPFS objects that hides IPFS-implementation details like layout
+objects (e.g. fanout and chunking).
+`,
+		Synopsis: `
+ipfs unixfs ls <path>...  - List directory contents for <path>...
+`,
+	},
+
+	Subcommands: map[string]*cmds.Command{
+		"ls": LsCmd,
+	},
+}

test/sharness/t0200-unixfs-ls.sh

+#!/bin/sh
+#
+# Copyright (c) 2014 Christian Couder
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test unixfs ls command"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+test_ls_cmd() {
+
+	test_expect_success "'ipfs add -r testData' succeeds" '
+		mkdir -p testData testData/d1 testData/d2 &&
+		echo "test" >testData/f1 &&
+		echo "data" >testData/f2 &&
+		echo "hello" >testData/d1/a &&
+		random 128 42 >testData/d1/128 &&
+		echo "world" >testData/d2/a &&
+		random 1024 42 >testData/d2/1024 &&
+		ipfs add -r testData >actual_add
+	'
+
+	test_expect_success "'ipfs add' output looks good" '
+		cat <<-\EOF >expected_add &&
+			added QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe testData/d1/128
+			added QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN testData/d1/a
+			added QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss testData/d1
+			added QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd testData/d2/1024
+			added QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL testData/d2/a
+			added QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy testData/d2
+			added QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH testData/f1
+			added QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M testData/f2
+			added QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj testData
+		EOF
+		test_cmp expected_add actual_add
+	'
+
+	test_expect_success "'ipfs unixfs ls <three dir hashes>' succeeds" '
+		ipfs unixfs ls QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss >actual_ls
+	'
+
+	test_expect_success "'ipfs unixfs ls <three dir hashes>' output looks good" '
+		cat <<-\EOF >expected_ls &&
+			QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj:
+			d1
+			d2
+			f1
+			f2
+
+			QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy:
+			1024
+			a
+
+			QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss:
+			128
+			a
+
+		EOF
+		test_cmp expected_ls actual_ls
+	'
+}
+
+
+# should work offline
+test_ls_cmd
+
+# should work online
+test_launch_ipfs_daemon
+test_ls_cmd
+test_kill_ipfs_daemon
+
+test_done