Merge pull request #26 from fission-suite/feat/add-ignore-rules

Feat/add ignore rules

filter.go

+package files
+
+import (
+	"os"
+
+	ignore "github.com/crackcomm/go-gitignore"
+)
+
+// Filter represents a set of rules for determining if a file should be included or excluded.
+// A rule follows the syntax for patterns used in .gitgnore files for specifying untracked files.
+// Examples:
+// foo.txt
+// *.app
+// bar/
+// **/baz
+// fizz/**
+type Filter struct {
+	// IncludeHidden - Include hidden files
+	IncludeHidden bool
+	// Rules - File filter rules
+	Rules *ignore.GitIgnore
+}
+
+// NewFilter creates a new file filter from a .gitignore file and/or a list of ignore rules.
+// An ignoreFile is a path to a file with .gitignore-style patterns to exclude, one per line
+// rules is an array of strings representing .gitignore-style patterns
+// For reference on ignore rule syntax, see https://git-scm.com/docs/gitignore
+func NewFilter(ignoreFile string, rules []string, includeHidden bool) (*Filter, error) {
+	var ignoreRules *ignore.GitIgnore
+	var err error
+	if ignoreFile == "" {
+		ignoreRules, err = ignore.CompileIgnoreLines(rules...)
+	} else {
+		ignoreRules, err = ignore.CompileIgnoreFileAndLines(ignoreFile, rules...)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &Filter{IncludeHidden: includeHidden, Rules: ignoreRules}, nil
+}
+
+// ShouldExclude takes an os.FileInfo object and applies rules to determine if its target should be excluded.
+func (filter *Filter) ShouldExclude(fileInfo os.FileInfo) (result bool) {
+	path := fileInfo.Name()
+	if !filter.IncludeHidden && isHidden(fileInfo) {
+		return true
+	}
+	return filter.Rules.MatchesPath(path)
+}

filter_test.go

+package files
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+type mockFileInfo struct {
+	os.FileInfo
+	name string
+}
+
+func (m *mockFileInfo) Name() string {
+	return m.name
+}
+
+var _ os.FileInfo = &mockFileInfo{}
+
+func TestFileFilter(t *testing.T) {
+	includeHidden := true
+	filter, err := NewFilter("", nil, includeHidden)
+	if err != nil {
+		t.Errorf("failed to create filter with empty rules")
+	}
+	if filter.IncludeHidden != includeHidden {
+		t.Errorf("new filter should include hidden files")
+	}
+	_, err = NewFilter("ignoreFileThatDoesNotExist", nil, false)
+	if err == nil {
+		t.Errorf("creating a filter without an invalid ignore file path should have failed")
+	}
+	tmppath, err := ioutil.TempDir("", "filter-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ignoreFilePath := filepath.Join(tmppath, "ignoreFile")
+	ignoreFileContents := []byte("a.txt")
+	if err := ioutil.WriteFile(ignoreFilePath, ignoreFileContents, 0666); err != nil {
+		t.Fatal(err)
+	}
+	filterWithIgnoreFile, err := NewFilter(ignoreFilePath, nil, false)
+	if err != nil {
+		t.Errorf("failed to create filter with ignore file")
+	}
+	if !filterWithIgnoreFile.ShouldExclude(&mockFileInfo{name: "a.txt"}) {
+		t.Errorf("filter should've excluded expected file from ignoreFile: %s", "a.txt")
+	}
+}

go.mod

 module github.com/ipfs/go-ipfs-files
 
-require golang.org/x/sys v0.0.0-20190302025703-b6889370fb10
+require (
+	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3
+	golang.org/x/sys v0.0.0-20190302025703-b6889370fb10
+)
 
 go 1.12

go.sum

+github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg=
+github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE=
 golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 h1:xQJI9OEiErEQ++DoXOHqEpzsGMrAv2Q2jyCpi7DmfpQ=
 golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

serialfile.go

@@ -11,16 +11,16 @@ import (
 // serialFile implements Node, and reads from a path on the OS filesystem.
 // No more than one file will be opened at a time.
 type serialFile struct {
-	path              string
-	files             []os.FileInfo
-	stat              os.FileInfo
-	handleHiddenFiles bool
+	path   string
+	files  []os.FileInfo
+	stat   os.FileInfo
+	filter *Filter
 }
 
 type serialIterator struct {
-	files             []os.FileInfo
-	handleHiddenFiles bool
-	path              string
+	files  []os.FileInfo
+	path   string
+	filter *Filter
 
 	curName string
 	curFile Node
@@ -28,8 +28,20 @@ type serialIterator struct {
 	err error
 }
 
-// TODO: test/document limitations
-func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) {
+// NewSerialFile takes a filepath, a bool specifying if hidden files should be included,
+// and a fileInfo and returns a Node representing file, directory or special file.
+func NewSerialFile(path string, includeHidden bool, stat os.FileInfo) (Node, error) {
+	filter, err := NewFilter("", nil, includeHidden)
+	if err != nil {
+		return nil, err
+	}
+	return NewSerialFileWithFilter(path, filter, stat)
+}
+
+// NewSerialFileWith takes a filepath, a filter for determining which files should be
+// operated upon if the filepath is a directory, and a fileInfo and returns a
+// Node representing file, directory or special file.
+func NewSerialFileWithFilter(path string, filter *Filter, stat os.FileInfo) (Node, error) {
 	switch mode := stat.Mode(); {
 	case mode.IsRegular():
 		file, err := os.Open(path)
@@ -44,7 +56,7 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) {
 		if err != nil {
 			return nil, err
 		}
-		return &serialFile{path, contents, stat, hidden}, nil
+		return &serialFile{path, contents, stat, filter}, nil
 	case mode&os.ModeSymlink != 0:
 		target, err := os.Readlink(path)
 		if err != nil {
@@ -72,7 +84,7 @@ func (it *serialIterator) Next() bool {
 
 	stat := it.files[0]
 	it.files = it.files[1:]
-	for !it.handleHiddenFiles && isHidden(stat) {
+	for it.filter.ShouldExclude(stat) {
 		if len(it.files) == 0 {
 			return false
 		}
@@ -87,7 +99,7 @@ func (it *serialIterator) Next() bool {
 	// recursively call the constructor on the next file
 	// if it's a regular file, we will open it as a ReaderFile
 	// if it's a directory, files in it will be opened serially
-	sf, err := NewSerialFile(filePath, it.handleHiddenFiles, stat)
+	sf, err := NewSerialFileWithFilter(filePath, it.filter, stat)
 	if err != nil {
 		it.err = err
 		return false
@@ -104,9 +116,9 @@ func (it *serialIterator) Err() error {
 
 func (f *serialFile) Entries() DirIterator {
 	return &serialIterator{
-		path:              f.path,
-		files:             f.files,
-		handleHiddenFiles: f.handleHiddenFiles,
+		path:   f.path,
+		files:  f.files,
+		filter: f.filter,
 	}
 }
 

serialfile_test.go

@@ -5,27 +5,30 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 )
 
-func isPathHidden(p string) bool {
+func isFullPathHidden(p string) bool {
 	return strings.HasPrefix(p, ".") || strings.Contains(p, "/.")
 }
 
 func TestSerialFile(t *testing.T) {
-	t.Run("Hidden", func(t *testing.T) { testSerialFile(t, true) })
-	t.Run("NotHidden", func(t *testing.T) { testSerialFile(t, false) })
+	t.Run("Hidden/NoFilter", func(t *testing.T) { testSerialFile(t, true, false) })
+	t.Run("Hidden/Filter", func(t *testing.T) { testSerialFile(t, true, true) })
+	t.Run("NotHidden/NoFilter", func(t *testing.T) { testSerialFile(t, false, false) })
+	t.Run("NotHidden/Filter", func(t *testing.T) { testSerialFile(t, false, true) })
 }
 
-func testSerialFile(t *testing.T, hidden bool) {
+func testSerialFile(t *testing.T, hidden, withIgnoreRules bool) {
 	tmppath, err := ioutil.TempDir("", "files-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmppath)
 
-	expected := map[string]string{
+	testInputs := map[string]string{
 		"1":      "Some text!\n",
 		"2":      "beep",
 		"3":      "",
@@ -38,8 +41,18 @@ func testSerialFile(t *testing.T, hidden bool) {
 		".8":     "",
 		".8/foo": "bla",
 	}
+	fileFilter, err := NewFilter("", []string{"9", "10"}, hidden)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if withIgnoreRules {
+		testInputs["9"] = ""
+		testInputs["9/b"] = "bebop"
+		testInputs["10"] = ""
+		testInputs["10/.c"] = "doowop"
+	}
 
-	for p, c := range expected {
+	for p, c := range testInputs {
 		path := filepath.Join(tmppath, p)
 		if c != "" {
 			continue
@@ -49,7 +62,7 @@ func testSerialFile(t *testing.T, hidden bool) {
 		}
 	}
 
-	for p, c := range expected {
+	for p, c := range testInputs {
 		path := filepath.Join(tmppath, p)
 		if c == "" {
 			continue
@@ -58,6 +71,22 @@ func testSerialFile(t *testing.T, hidden bool) {
 			t.Fatal(err)
 		}
 	}
+	expectedHiddenPaths := make([]string, 0, 4)
+	expectedRegularPaths := make([]string, 0, 6)
+	for p := range testInputs {
+		path := filepath.Join(tmppath, p)
+		stat, err := os.Stat(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !fileFilter.ShouldExclude(stat) {
+			if isFullPathHidden(path) {
+				expectedHiddenPaths = append(expectedHiddenPaths, p)
+			} else {
+				expectedRegularPaths = append(expectedRegularPaths, p)
+			}
+		}
+	}
 
 	stat, err := os.Stat(tmppath)
 	if err != nil {
@@ -65,12 +94,17 @@ func testSerialFile(t *testing.T, hidden bool) {
 	}
 
 	sf, err := NewSerialFile(tmppath, hidden, stat)
+	if withIgnoreRules {
+		sf, err = NewSerialFileWithFilter(tmppath, fileFilter, stat)
+	}
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer sf.Close()
 
 	rootFound := false
+	actualRegularPaths := make([]string, 0, len(expectedRegularPaths))
+	actualHiddenPaths := make([]string, 0, len(expectedHiddenPaths))
 	err = Walk(sf, func(path string, nd Node) error {
 		defer nd.Close()
 
@@ -85,16 +119,23 @@ func testSerialFile(t *testing.T, hidden bool) {
 			rootFound = true
 			return nil
 		}
-
-		if !hidden && isPathHidden(path) {
+		if isFullPathHidden(path) {
+			actualHiddenPaths = append(actualHiddenPaths, path)
+		} else {
+			actualRegularPaths = append(actualRegularPaths, path)
+		}
+		if !hidden && isFullPathHidden(path) {
 			return fmt.Errorf("found a hidden file")
 		}
+		if fileFilter.Rules.MatchesPath(path) {
+			return fmt.Errorf("found a file that should be excluded")
+		}
 
-		data, ok := expected[path]
+		data, ok := testInputs[path]
 		if !ok {
 			return fmt.Errorf("expected something at %q", path)
 		}
-		delete(expected, path)
+		delete(testInputs, path)
 
 		switch nd := nd.(type) {
 		case *Symlink:
@@ -117,10 +158,16 @@ func testSerialFile(t *testing.T, hidden bool) {
 	if !rootFound {
 		t.Fatal("didn't find the root")
 	}
-	for p := range expected {
-		if !hidden && isPathHidden(p) {
-			continue
+	for _, regular := range expectedRegularPaths {
+		if idx := sort.SearchStrings(actualRegularPaths, regular); idx < 0 {
+			t.Errorf("missed regular path %q", regular)
+		}
+	}
+	if hidden && len(actualHiddenPaths) != len(expectedHiddenPaths) {
+		for _, missing := range expectedHiddenPaths {
+			if idx := sort.SearchStrings(actualHiddenPaths, missing); idx < 0 {
+				t.Errorf("missed hidden path %q", missing)
+			}
 		}
-		t.Errorf("missed %q", p)
 	}
 }