params.go

package coreindex

import (
	"errors"
	"fmt"
	"io"
	"strings"

	//"time"
	"bytes"
	"encoding/xml"
	"os"
	"path/filepath"

	"github.com/facebookgo/atomicfile"

	idxconfig "gitlab.dms3.io/is/go-idx-config"
	idxis "gitlab.dms3.io/is/go-idx-engine"
)

type makeindexOpts struct {
	key string
	enc string
}

func IsKindConfigured(value interface{}, kind string) (found bool, err error) {

	if iconf, ok := value.(*idxconfig.IdxConfig); ok {
		a := iconf.Metadata.Kind

		for i := range a {
			if a[i].Name == kind {
				for f := range a[i].Field {
					if a[i].Field[f] != "" {
						found = true
					}
				}
			}
		}
	}
	return
}

func MakeParams(cfg interface{}, path, kind string) (filename string, err error) {
	if filename, err = ParamsLocalFile(kind); err != nil {
		return
	}

	//
	// don't overwrite existing params file to protect the indexer.
	//
	// once index is created, we don't want to modify structure.
	// otherwise the search engine will misbehave with incorrect
	// query results or worse, the engine may crash.
	//
	// modifying index structure will renumber terms like metadata
	// and fields.
	//
	if !PathExists(filename) {
		_, err = writeParamFile(filename, cfg, kind)
	}
	return
}

func MakeRepo(cfg interface{}, repoPath, kind string) (cp, reponame string, ctime int64, err error) {
	cp, err = makeRepoSubDirs(repoPath)
	//reponame, ctime, err = MakeIndex(repoPath, kind, cp)
	reponame, ctime, err = MakeIndex(repoPath, kind, cp)
	return
}

func MakeIndex(rpath, kind, cp string) (reponame string, ctime int64, err error) {

	var params, stopwords string = "", ""

	// assumes params file is created already
	if params, err = ParamsLocalFile(kind); err != nil {
		return
	}

	// optional, if kind/stopwords exists or ''
	if stopwords, err = StopwordsLocalFile(kind); err != nil {
		return
	}

	var is idxis.InfoSpace
	var a_app string = "dms3"                // app creating this infospace (index metadata)
	var a_space string = kind                // kind of inforspace (used in index path, field)
	var a_server string = ""                 // N/A, not used
	var a_command string = "IndriBuildIndex" // for legacy command parsing
	var a_index_path string = rpath          // path to index
	var a_corpus_path string = cp            // path to corpus documents
	var a_corpus_class string = "html"       // document class
	var a_build_param string = params        // path to params file
	var a_stop_param string = stopwords      // path to stop words file

	// create index repository
	is, err = idxis.NewInfoSpace(a_app,
		a_space,
		a_server,
		a_command,
		a_index_path,
		a_corpus_path,
		a_corpus_class,
		a_build_param,
		a_stop_param)

	if err != nil {
		fmt.Printf("error: %v\n", err)
		return
	}
	defer idxis.DeleteInfoSpace(is)
	//defer func(a idxis.InfoSpace) {
	//    fmt.Printf("deferred delete param: %x\n", a)
	//    idxis.DeleteInfoSpace(a)
	//}(is)

	// attach to new index, this will initialize on disk structures
	_, err = is.InfoSpaceAttach()
	if err != nil {
		return
	}

	r_appName, err := is.InfoSpaceGetApp()
	if err != nil {
		return
	}

	r_isName, err := is.InfoSpaceGetName()
	if err != nil {
		return
	}

	r_window, err := is.InfoSpaceGetWindow()
	if err != nil {
		return
	}

	ctime = r_window

	window := fmt.Sprintf("w%d", r_window) // seconds since Unix epoch
	reponame = filepath.Join(r_appName+"-"+r_isName, window)

	// detach from the index.
	// now ready for document index and query functions.
	_, err = is.InfoSpaceDetach()
	if err != nil {
		return
	}

	return
}

func MakeDoc(iconf idxconfig.IdxConfig, kind string) (string, error) {

	var found bool = false

	var b []byte
	w := bytes.NewBuffer(b)

	a := iconf.Metadata.Kind
	for i := range a {

		if a[i].Name == kind {
			found = true

			enc := xml.NewEncoder(w)
			enc.Indent("  ", "    ")
			var sel xml.StartElement
			var snm xml.Name
			snm.Space = ""
			snm.Local = kind
			sel.Name = snm

			if err := enc.EncodeToken(sel); err != nil {
				fmt.Printf("error: %v\n", err)
			}
			for f := range a[i].Field {
				var el xml.StartElement
				var en xml.Name
				en.Space = ""
				en.Local = strings.ToLower(a[i].Field[f])
				el.Name = en
				if err := enc.EncodeElement("", el); err != nil {
					fmt.Printf("error: %v\n", err)
				}
			}
			if err := enc.EncodeToken(sel.End()); err != nil {
				fmt.Printf("error: %v\n", err)
			}
			if err := enc.Flush(); err != nil {
				fmt.Printf("error: %v\n", err)
			}
		}
	}

	if found {
		return w.String(), nil
	} else {
		return "", errors.New("specified kind is not found, please use \"dms3 index config\" command to configure.")
	}
}

//
// writeParamFile writes the index repository parameters from `cfg` into `filename`.
// configuration file property to index paramater file element mappings can be
// summarized as follows:
//
// index and corpus parameter configuration
//   params[index] = cfg Indexer.Path * code overrides configured value *
//   params[corpus][annotation] = cfg Indexer.Corpus.Annotation * not used *
//   params[corpus] = cfg Indexer.Corpus
//   params[corpus][path] = cfg Indexer.Corpus.Path * code overrides configured value *
//   params[corpus][class] = cfg Indexer.Corpus.Class
//   params[corpus][metadata] = cfg Indexer.Corpus.Metadata * not used *
//
// optional parameter configuration
//   params[memory] = cfg Indexer.Memory
//   params[stemmer][name] = cfg Indexer.Stemmer
//   params[normalize] = cfg Indexer.Normalize
//   params[stopper][word] = cfg Indexer.Stopper[i]
//
// metadata and field parameter configuration is hard coded
//
// document kind-specific field parameter configuration
//   params[field][name][f] = cfg Metadata.Kind[i][f]
//
// note: the infospace interface can override some parameters at time of
// index creation (see MakeIndex).
//
// TODO: remove from index configuration:
//  Indexer.Corpus:
//      annotations not used
//      path is overriden (computed)
//      metadata not used
//
//  Indexer.Path:
//      path is overriden (computed)
//
//  Indexer.Stopper: []
//      is overriden, or complemented by global stopwords file
//
func writeParamFile(filename string, cfg interface{}, kind string) (bool, error) {
	err := os.MkdirAll(filepath.Dir(filename), 0775)
	if err != nil {
		return false, err
	}

	f, err := atomicfile.New(filename, 0660)
	if err != nil {
		return false, err
	}
	defer f.Close()

	return encode(f, cfg, kind)
}

// given repo params file, create repo subfolders
// cfg path parameters are ignored for these
//	  - Indexer.Corpus.Path
//	  - Indexer.Corpus.Metadata
func makeRepoSubDirs(repopath string) (c string, err error) {

	// index folders under repopath are managed by infospace
	c = filepath.Join(repopath, "corpus")

	//fmt.Printf("corpus: %v\n", c)

	if err = os.MkdirAll(repopath, 0775); err != nil {
		return
	}
	if err = os.Mkdir(c, 0755); err != nil {
		return
	}
	return
}

func encode(w io.Writer, value interface{}, kind string) (found bool, err error) {

	// encode the index parameters file from configured properties
	enc := xml.NewEncoder(w)
	enc.Indent("  ", "    ")

	var sel xml.StartElement
	var snm xml.Name
	snm.Space = ""
	snm.Local = "parameters"
	sel.Name = snm
	if err := enc.EncodeToken(sel); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	if found, err = makeIndexPathCorpusParams(value, enc); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	if found, err = makeIndexMetadataParams(value, enc); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	if found, err = makeIndexFieldsParams(value, enc, kind); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	if found, err = makeIndexMemStemNormStopParams(value, enc); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	if err := enc.EncodeToken(sel.End()); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	if err := enc.Flush(); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}
	return found, err
}

// index and corpus parameter configuration
func makeIndexPathCorpusParams(value interface{}, enc *xml.Encoder) (found bool, err error) {

	if iconf, ok := value.(*idxconfig.IdxConfig); ok {
		ix := iconf.Indexer // cfg Indexer object

		/*
			        if ix.Path != "" {    // cfg Indexer.Path string
						found = true

			            // params[index] = cfg Indexer.Path
						var el xml.StartElement
						var en xml.Name
						en.Space = ""
						en.Local = "index"
						el.Name = en
						if err := enc.EncodeElement(strings.ToLower(ix.Path),el); err != nil {
							fmt.Printf("error: %v\n", err)
							found = false
							return found, err
						}
					} else {
						err = errors.New("missing index path parameters, please use \"dms3 index config\" command to verify configure.")
						found = false
						return found, err
					}
		*/

		if ix.Corpus != (idxconfig.Corpus{}) { // cfg Indexer.Corpus
			found = true

			// params file does not use cfg Indexer.Corpus.Annotation

			// params[corpus] = cfg Indexer.Corpus
			var cel xml.StartElement
			var cen xml.Name
			cen.Space = ""
			cen.Local = "corpus"
			cel.Name = cen
			if err := enc.EncodeToken(cel); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}

			// params[corpus][path] = cfg Indexer.Corpus.Path
			/*
				if ix.Corpus.Path != "" {
					var cpel xml.StartElement
					var cpen xml.Name
					cpen.Space = ""
					cpen.Local = "path"
					cpel.Name = cpen
					if err := enc.EncodeElement(strings.ToLower(ix.Corpus.Path),cpel); err != nil {
						fmt.Printf("error: %v\n", err)
						found = false
						return found, err
					}
				}
			*/

			// params[corpus][class] = cfg Indexer.Corpus.Class
			if ix.Corpus.Class != "" {
				var ccel xml.StartElement
				var ccen xml.Name
				ccen.Space = ""
				ccen.Local = "class"
				ccel.Name = ccen
				if err := enc.EncodeElement(strings.ToLower(ix.Corpus.Class), ccel); err != nil {
					fmt.Printf("error: %v\n", err)
					found = false
					return found, err
				}
			}

			// params[corpus][metadata] = cfg Indexer.Corpus.Metadata
			/*
				if ix.Corpus.Metadata != "" {
					var cmel xml.StartElement
					var cmen xml.Name
					cmen.Space = ""
					cmen.Local = "metadata"
					cmel.Name = cmen
					if err := enc.EncodeElement(strings.ToLower(ix.Corpus.Metadata),cmel); err != nil {
						fmt.Printf("error: %v\n", err)
						found = false
						return found, err
					}
				}
			*/

			if err := enc.EncodeToken(cel.End()); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}
		} else {
			err = errors.New("missing index corpus parameters, please use \"dms3 index config\" command to verify configure.")
			found = false
			return found, err
		}
	}

	return found, err
}

// optional parameter configuration
func makeIndexMemStemNormStopParams(value interface{}, enc *xml.Encoder) (found bool, err error) {

	if iconf, ok := value.(*idxconfig.IdxConfig); ok {
		var n1, n2 xml.CharData
		var c1 xml.Comment
		var n1b, n2b, c1b []byte
		n1s := bytes.NewBuffer(n1b)
		n2s := bytes.NewBuffer(n2b)
		c1s := bytes.NewBuffer(c1b)
		n1s.WriteString("\n")
		n2s.WriteString("\n\n")
		c1s.WriteString(" optional index parameters ")
		n1 = n1s.Bytes()
		n2 = n2s.Bytes()
		c1 = c1s.Bytes()

		if err := enc.EncodeToken(n2); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}
		if err := enc.EncodeToken(c1); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}
		if err := enc.EncodeToken(n1); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}

		ix := iconf.Indexer

		// params[memory] = cfg Indexer.Memory
		if ix.Memory != "" {
			var imel xml.StartElement
			var imen xml.Name
			imen.Space = ""
			imen.Local = "memory"
			imel.Name = imen
			if err := enc.EncodeElement(strings.ToLower(ix.Memory), imel); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}
		}

		if ix.Stemmer != "" {
			var isel xml.StartElement
			var isen xml.Name
			isen.Space = ""
			isen.Local = "stemmer"
			isel.Name = isen
			if err := enc.EncodeToken(isel); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}

			// params[stemmer][name] = cfg Indexer.Stemmer
			var isnel xml.StartElement
			var isnen xml.Name
			isnen.Space = ""
			isnen.Local = "name"
			isnel.Name = isnen
			if err := enc.EncodeElement(strings.ToLower(ix.Stemmer), isnel); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}

			if err := enc.EncodeToken(isel.End()); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}
		}

		// params[normalize] = cfg Indexer.Normalize
		var inel xml.StartElement
		var inen xml.Name
		inen.Space = ""
		inen.Local = "normalize"
		inel.Name = inen
		if err := enc.EncodeElement(ix.Normalize, inel); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}

		if len(ix.Stopper) > 0 {
			found = true

			var istel xml.StartElement
			var isten xml.Name
			isten.Space = ""
			isten.Local = "stopper"
			istel.Name = isten
			if err := enc.EncodeToken(istel); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}

			for swd := range ix.Stopper { // cfg Indexer.Stopper
				// params[stopper][word] = cfg Indexer.Stopper[i]
				var swel xml.StartElement
				var swen xml.Name
				swen.Space = ""
				swen.Local = "word"
				swel.Name = swen
				if err := enc.EncodeElement(strings.ToLower(ix.Stopper[swd]), swel); err != nil {
					fmt.Printf("error: %v\n", err)
					found = false
					return found, err
				}
			}

			if err := enc.EncodeToken(istel.End()); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}
		}
	}

	return found, err
}

// metadata and field parameter configuration
func makeIndexMetadataParams(value interface{}, enc *xml.Encoder) (found bool, err error) {

	var n1, n2 xml.CharData
	var c, c1, c2, c3, c4 xml.Comment
	var n1b, n2b, c1b, c2b, c3b, c4b []byte
	n1s := bytes.NewBuffer(n1b)
	n2s := bytes.NewBuffer(n2b)
	c1s := bytes.NewBuffer(c1b)
	c2s := bytes.NewBuffer(c2b)
	c3s := bytes.NewBuffer(c3b)
	c4s := bytes.NewBuffer(c4b)
	n1s.WriteString("\n")
	n2s.WriteString("\n\n")
	c1s.WriteString(" read-only life-cycle [system] metadata fields ")
	c2s.WriteString(" read-write life-cycle [document] metadata fields ")
	c3s.WriteString(" start of [document] kind common metadata fields ")
	c4s.WriteString(" start of [document] kind specific metadata fields ")
	n1 = n1s.Bytes()
	n2 = n2s.Bytes()
	c1 = c1s.Bytes()
	c2 = c2s.Bytes()
	c3 = c3s.Bytes()
	c4 = c4s.Bytes()

	// index metadata, forward, and reverse fields are hard coded
	rs := []string{
		// read-only [system] metadata
		"odmver",
		"schver",
		"base-time",
		"curr-epoch",
		"prev-epoch",
		"max-areas",
		"max-cats",
		"app",
		// read-write [application] metadata fields in the index paramaters file
		"docno",
		"docver",
		// start of template specific fields - generated elsewhere
	}

	found = true

	var imel xml.StartElement
	var imnm xml.Name
	imnm.Space = ""
	imnm.Local = "metadata"
	imel.Name = imnm
	if err := enc.EncodeToken(imel); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	for i := range rs {
		switch strings.ToLower(rs[i]) {
		case "odmver", "docno":
			if err := enc.EncodeToken(n2); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}
			if strings.ToLower(rs[i]) == "odmver" {
				c = c1
			} else {
				c = c2
			}
			if err := enc.EncodeToken(c); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}
			if err := enc.EncodeToken(n1); err != nil {
				fmt.Printf("error: %v\n", err)
				found = false
				return found, err
			}
		}
		var fel xml.StartElement
		var fen xml.Name
		fen.Space = ""
		fen.Local = "forward"
		fel.Name = fen
		if err := enc.EncodeElement(strings.ToLower(rs[i]), fel); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}
	}
	if err := enc.EncodeToken(n2); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}
	if err := enc.EncodeToken(c3); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}
	if err := enc.EncodeToken(n1); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	for i := range rs {
		var bel xml.StartElement
		var ben xml.Name
		ben.Space = ""
		ben.Local = "backward"
		bel.Name = ben
		if err := enc.EncodeElement(strings.ToLower(rs[i]), bel); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}
	}

	for i := range rs {

		var fdel xml.StartElement
		var fdnm xml.Name
		fdnm.Space = ""
		fdnm.Local = "field"
		fdel.Name = fdnm
		if err := enc.EncodeToken(fdel); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}

		var nel xml.StartElement
		var nnm xml.Name
		nnm.Space = ""
		nnm.Local = "name"
		nel.Name = nnm
		if err := enc.EncodeElement(strings.ToLower(rs[i]), nel); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}

		if err := enc.EncodeToken(fdel.End()); err != nil {
			fmt.Printf("error: %v\n", err)
			found = false
			return found, err
		}
	}

	if err := enc.EncodeToken(imel.End()); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}

	if err := enc.EncodeToken(n2); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}
	if err := enc.EncodeToken(c4); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}
	if err := enc.EncodeToken(n1); err != nil {
		fmt.Printf("error: %v\n", err)
		found = false
		return found, err
	}
	return found, err
}

// kind-specific field parameter configuration
func makeIndexFieldsParams(value interface{}, enc *xml.Encoder, kind string) (found bool, err error) {

	if iconf, ok := value.(*idxconfig.IdxConfig); ok {
		a := iconf.Metadata.Kind // cfg Metadata.Kind

		for i := range a {

			if a[i].Name == kind { // cfg specific Metadata.Kind
				found = true

				var kfel xml.StartElement
				var kfnm xml.Name
				kfnm.Space = ""
				kfnm.Local = "field"
				kfel.Name = kfnm
				if err := enc.EncodeToken(kfel); err != nil {
					fmt.Printf("error: %v\n", err)
					found = false
					return found, err
				}

				var knel xml.StartElement
				var knnm xml.Name
				knnm.Space = ""
				knnm.Local = "name"
				knel.Name = knnm
				if err := enc.EncodeElement(strings.ToLower(kind), knel); err != nil {
					fmt.Printf("error: %v\n", err)
					found = false
					return found, err
				}

				if err := enc.EncodeToken(kfel.End()); err != nil {
					fmt.Printf("error: %v\n", err)
					found = false
					return found, err
				}

				// params[field][name][f] = cfg Metadata.Kind[i][f]
				for f := range a[i].Field {

					var fdel xml.StartElement
					var fdnm xml.Name
					fdnm.Space = ""
					fdnm.Local = "field"
					fdel.Name = fdnm
					if err := enc.EncodeToken(fdel); err != nil {
						fmt.Printf("error: %v\n", err)
						found = false
						return found, err
					}

					var nel xml.StartElement
					var nnm xml.Name
					nnm.Space = ""
					nnm.Local = "name"
					nel.Name = nnm
					if err := enc.EncodeElement(strings.ToLower(a[i].Field[f]), nel); err != nil {
						fmt.Printf("error: %v\n", err)
						found = false
						return found, err
					}

					if err := enc.EncodeToken(fdel.End()); err != nil {
						fmt.Printf("error: %v\n", err)
						found = false
						return found, err
					}
				}
			}
		}
	}

	return found, err
}