.gitignore

+cid-fuzz.zip

.gitlab-ci.yml

+stages:
+  - build
+  - test
+
+variables:
+  BUILD_DIR: "/tmp/$CI_CONCURRENT_PROJECT_ID"
+
+before_script:
+  - mkdir -p $BUILD_DIR/src
+  - cd $BUILD_DIR/src
+  - if [ -d $CI_PROJECT_DIR ]
+  - then
+  -    echo "soft link $CI_PROJECT_DIR exists"
+  - else
+  -    echo "creating soft link $CI_PROJECT_DIR"
+  -    ln -s $CI_PROJECT_DIR
+  - fi
+  - cd $CI_PROJECT_DIR
+
+build:
+  stage: build
+  tags:
+    - testing
+  script:
+    - echo $CI_JOB_STAGE
+    - go build
+
+test:
+  stage: test
+  tags:
+    - testing
+  script:
+    - echo $CI_JOB_STAGE
+    - go test -cover
+  coverage: '/coverage: \d+.\d+% of statements/'

config.go

+// package idxconfig implements the dms3fs idxconfig file datastructures and utilities.
+package idxconfig
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mitchellh/go-homedir"
+)
+
+// IdxConfig is used to load dms3fs index parameters idxconfig files.
+type IdxConfig struct {
+        Indexer   Indexer
+        Metadata  Metadata
+        Retriever Retriever
+}
+
+const (
+	// DefaultPathName is the default idxconfig dir name
+	DefaultPathName = ".dms3-fs/index"
+	// DefaultPathRoot is the path to the default idxconfig dir location.
+	DefaultPathRoot = "~/" + DefaultPathName
+	// DefaultIdxConfigFile is the filename of the configuration file
+	DefaultIdxConfigFile = "idxconfig"
+	// EnvDir is the environment variable used to change the path root.
+	EnvDir = "DMS3_PATH"	// must be relative
+)
+
+// PathRoot returns the default configuration root directory
+func PathRoot() (string, error) {
+	dir := os.Getenv(EnvDir)
+	var err error
+	if len(dir) == 0 {
+		dir, err = homedir.Expand(DefaultPathRoot)
+	}
+	return dir, err
+}
+
+// Path returns the path `extension` relative to the configuration root. If an
+// empty string is provided for `configroot`, the default root is used.
+func Path(configroot, extension string) (string, error) {
+	if len(configroot) == 0 {
+		dir, err := PathRoot()
+		if err != nil {
+			return "", err
+		}
+		return filepath.Join(dir, extension), nil
+
+	}
+	return filepath.Join(configroot, extension), nil
+}
+
+// Filename returns the configuration file path given a configuration root
+// directory. If the configuration root directory is empty, use the default one
+func Filename(configroot string) (string, error) {
+	return Path(configroot, DefaultIdxConfigFile)
+}
+
+// HumanOutput gets a idxconfig value ready for printing
+func HumanOutput(value interface{}) ([]byte, error) {
+	s, ok := value.(string)
+	if ok {
+		return []byte(strings.Trim(s, "\n")), nil
+	}
+	return Marshal(value)
+}
+
+// Marshal configuration with JSON
+func Marshal(value interface{}) ([]byte, error) {
+	// need to prettyprint, hence MarshalIndent, instead of Encoder
+	return json.MarshalIndent(value, "", "  ")
+}
+
+func FromMap(v map[string]interface{}) (*IdxConfig, error) {
+	buf := new(bytes.Buffer)
+	if err := json.NewEncoder(buf).Encode(v); err != nil {
+		return nil, err
+	}
+	var conf IdxConfig
+	if err := json.NewDecoder(buf).Decode(&conf); err != nil {
+		return nil, fmt.Errorf("failure to decode idxconfig: %s", err)
+	}
+	return &conf, nil
+}
+
+func ToMap(conf *IdxConfig) (map[string]interface{}, error) {
+	buf := new(bytes.Buffer)
+	if err := json.NewEncoder(buf).Encode(conf); err != nil {
+		return nil, err
+	}
+	var m map[string]interface{}
+	if err := json.NewDecoder(buf).Decode(&m); err != nil {
+		return nil, fmt.Errorf("failure to decode idxconfig: %s", err)
+	}
+	return m, nil
+}

go.mod

+module gitlab.dms3.io/is/go-idx-config
+
+go 1.15
+
+require (
+	github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5
+	github.com/mitchellh/go-homedir v1.1.0
+	gitlab.dms3.io/dms3/go-dms3-util v0.0.2
+)

go.sum

+github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A=
+github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
+github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
+github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg=
+github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+gitlab.dms3.io/dms3/go-dms3-util v0.0.2 h1:xXHeLaht5szd3QPdQp5KcfmXbIyRvRPSdQnMkJPRgPo=
+gitlab.dms3.io/dms3/go-dms3-util v0.0.2/go.mod h1:5hPwxzo5zK4NeHE/anWIQGHcIoG7aTl9/Pp0j2zg0l8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

indexer.go

+package idxconfig
+
+// Indexer stores index repository server configuration.
+type Indexer struct {
+	Path		string 	// path to index, ex: repo/base
+	Memory		string
+	Stemmer		string
+	Normalize	bool
+	Stopper		[]string
+	Corpus 		Corpus
+    MaxDocs     string
+}
+
+type Corpus struct {
+    Path		string	// path to repository, ex: repo/base/corpus
+    Class		string
+}

init.go

+package idxconfig
+
+import (
+	//"encoding/base64"
+	//"errors"
+	//"fmt"
+	"io"
+	//"time"
+
+)
+
+func Init(out io.Writer) (*IdxConfig, error) {
+
+	//datastore := DefaultDatastoreConfig()
+
+	iconf := &IdxConfig{
+		Indexer: Indexer{
+			Path: 		"", // path to index, ex: repo/base
+			Memory: 	"100M",
+			Stemmer: 	"krovetz",
+			Normalize: 	true,
+			Stopper: 	[]string{
+				"a",
+				"an",
+				"the",
+				"as",
+			},
+			Corpus: 	Corpus{
+				Path: 			"", // path to repository, ex: repo/base/corpus
+				Class:			"html",
+			},
+            MaxDocs:    "100M",     // max # of doc per index
+		},
+		Metadata: Metadata{
+    		Kind: []Kind{{
+				Name: "blog",
+				Field: []string{
+					"About",
+	            	"Address",
+	            	"Affiliation",
+	            	"Author",
+	            	"Brand",
+	            	"Citation",
+	            	"Description",
+	            	"Email",
+	            	"Headline",
+	            	"Keywords",
+	            	"Language",
+	            	"Name",
+	            	"Telephone",
+	            	"Version",
+					},
+				},
+			},
+  		},
+		Retriever: Retriever{
+			MaxResultCount: 100,
+		},
+	}
+
+	return iconf, nil
+}

metadata.go

+package idxconfig
+
+// Metadata stores index repository metadata configuration.
+type Metadata struct {
+    Kind 		[]Kind
+}
+
+type Kind struct {
+	Name		string
+	Field		[]string
+}
+
+/*
+Metadata: Metadata{
+	Field: [
+		// read-only [system] metadata
+		"odmver",		// ODM version
+		"schver",		// schema version
+		"base-time", 	// window start
+        "curr-epoch", 	// current epoch
+		"prev-epoch", 	// previous epoch
+        "max-areas", 	// maximum number of areas
+		"max-cats", 	// maximum number of categories
+		"app", 			// application or service name
+		// read-write [application] metadata fields in the index paramaters file
+		"docno", 		// document number
+		"docver", 		// document version
+		// start of template specific fields - generated elsewhere
+	],
+	Forward: [
+		// read-only [system] metadata
+		"odmver",		// ODM version
+		"schver",		// schema version
+		"base-time", 	// window start
+        "curr-epoch", 	// current epoch
+		"prev-epoch", 	// previous epoch
+		"max-areas", 	// maximum number of areas
+		"max-cats", 	// maximum number of categories
+		"app", 			// application or service name
+		// read-write [application] metadata fields in the index paramaters file
+		"docno", 		// document number
+		"docver", 		// document version
+	],
+	Backword: [
+		// read-only [system] metadata
+		"odmver",		// ODM version
+		"schver",		// schema version
+		"base-time", 	// window start
+        "curr-epoch", 	// current epoch
+		"prev-epoch", 	// previous epoch
+		"max-areas", 	// maximum number of areas
+		"max-cats", 	// maximum number of categories
+		"app", 			// application or service name
+		// read-write [application] metadata fields in the index paramaters file
+		"docno", 		// document number
+		"docver", 		// document version
+	],
+},
+*/

retriever.go

+package idxconfig
+
+// Retriever stores the query retriever configuration.
+type Retriever struct {
+	MaxResultCount      uint16 // maximum query result count
+}

serialize/serialize.go

+package idxrepo
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/facebookgo/atomicfile"
+	util "gitlab.dms3.io/dms3/go-dms3-util"
+	idxconfig "gitlab.dms3.io/is/go-idx-config"
+)
+
+// ReadConfigFile reads the idxconfig from `filename` into `cfg`.
+func ReadConfigFile(filename string, cfg interface{}) error {
+	f, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if err := json.NewDecoder(f).Decode(cfg); err != nil {
+		return fmt.Errorf("failure to decode idxconfig: %s", err)
+	}
+	return nil
+}
+
+// WriteConfigFile writes the idxconfig from `cfg` into `filename`.
+func WriteConfigFile(filename string, cfg interface{}) error {
+	err := os.MkdirAll(filepath.Dir(filename), 0775)
+	if err != nil {
+		return err
+	}
+
+	f, err := atomicfile.New(filename, 0660)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return encode(f, cfg)
+}
+
+// encode configuration with JSON
+func encode(w io.Writer, value interface{}) error {
+	// need to prettyprint, hence MarshalIndent, instead of Encoder
+	buf, err := idxconfig.Marshal(value)
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(buf)
+	return err
+}
+
+// Load reads given file and returns the read idxconfig, or error.
+func Load(filename string) (*idxconfig.IdxConfig, error) {
+	// if nothing is there, fail. User must run 'dms3fs init'
+	if !util.FileExists(filename) {
+		return nil, errors.New("dms3fs not initialized, please run 'dms3fs init'")
+	}
+
+	var cfg idxconfig.IdxConfig
+	err := ReadConfigFile(filename, &cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg, err
+}

serialize/serialize_test.go

+package idxrepo
+
+import (
+	"os"
+	"runtime"
+	"testing"
+
+	idxconfig "gitlab.dms3.io/is/go-idx-config"
+)
+
+func TestConfig(t *testing.T) {
+	const filename = ".dms3config"
+	cfgWritten := new(idxconfig.IdxConfig)
+	cfgWritten.Identity.PeerID = "faketest"
+
+	err := WriteConfigFile(filename, cfgWritten)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfgRead, err := Load(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfgWritten.Identity.PeerID != cfgRead.Identity.PeerID {
+		t.Fatal()
+	}
+	st, err := os.Stat(filename)
+	if err != nil {
+		t.Fatalf("cannot stat idxconfig file: %v", err)
+	}
+
+	if runtime.GOOS != "windows" { // see https://golang.org/src/os/types_windows.go
+		if g := st.Mode().Perm(); g&0117 != 0 {
+			t.Fatalf("idxconfig file should not be executable or accessible to world: %v", g)
+		}
+	}
+}