diff --git a/.travis.yml b/.travis.yml index b86c124dd10e1ca4bad8b2484237f992724e929a..60b9cae5a8217c1bdc7dfc941e7b9ea417b6edcf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ os: language: go go: - - 1.11.x + - 1.12.x env: global: diff --git a/go.mod b/go.mod index 1f573bf5b65e9254e0efd745bfdb92e9a17be2d8..3d2cb74310f10d3f2c46bc45edc2a94e6c626199 100644 --- a/go.mod +++ b/go.mod @@ -9,3 +9,5 @@ require ( golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) + +go 1.12 diff --git a/mount/mount.go b/mount/mount.go index 1f97270e221b208f00ab0d486a1f60259289a4f0..7c32a8271b24f773d77a9cdcafb2a1305fdbcc5c 100644 --- a/mount/mount.go +++ b/mount/mount.go @@ -224,7 +224,6 @@ func (d *Datastore) Delete(key ds.Key) error { func (d *Datastore) Query(master query.Query) (query.Results, error) { childQuery := query.Query{ Prefix: master.Prefix, - Limit: master.Limit, Orders: master.Orders, KeysOnly: master.KeysOnly, ReturnExpirations: master.ReturnExpirations, @@ -254,7 +253,7 @@ func (d *Datastore) Query(master query.Query) (query.Results, error) { queries.addResults(mount, results) } - qr := query.ResultsFromIterator(childQuery, query.Iterator{ + qr := query.ResultsFromIterator(master, query.Iterator{ Next: queries.next, Close: queries.close, }) @@ -269,8 +268,8 @@ func (d *Datastore) Query(master query.Query) (query.Results, error) { qr = query.NaiveOffset(qr, master.Offset) } - if childQuery.Limit > 0 { - qr = query.NaiveLimit(qr, childQuery.Limit) + if master.Limit > 0 { + qr = query.NaiveLimit(qr, master.Limit) } return qr, nil diff --git a/query/filter.go b/query/filter.go index 501414d3b89e52492b9971bf6c89c0b46aa7ee00..1935c4888ab74cfb94ed859c7388cddc95290346 100644 --- a/query/filter.go +++ b/query/filter.go @@ -57,6 +57,10 @@ func (f FilterValueCompare) Filter(e Entry) bool { } } +func (f FilterValueCompare) String() string { + return fmt.Sprintf("VALUE %s %q", f.Op, string(f.Value)) +} + type FilterKeyCompare struct { Op Op Key string @@ -81,6 +85,10 @@ func (f FilterKeyCompare) Filter(e Entry) bool { } } +func (f FilterKeyCompare) String() string { + return fmt.Sprintf("KEY %s %q", f.Op, f.Key) +} + type FilterKeyPrefix struct { Prefix string } @@ -88,3 +96,7 @@ type FilterKeyPrefix struct { func (f FilterKeyPrefix) Filter(e Entry) bool { return strings.HasPrefix(e.Key, f.Prefix) } + +func (f FilterKeyPrefix) String() string { + return fmt.Sprintf("PREFIX(%q)", f.Prefix) +} diff --git a/query/order.go b/query/order.go index 5e0acf1bfac23b49a2f79c5e51d089943d987bc1..19931557e45c896aad3143a1ebbf5d7071950d38 100644 --- a/query/order.go +++ b/query/order.go @@ -18,6 +18,10 @@ func (o OrderByFunction) Compare(a, b Entry) int { return o(a, b) } +func (OrderByFunction) String() string { + return "FN" +} + // OrderByValue is used to signal to datastores they should apply internal // orderings. type OrderByValue struct{} @@ -26,6 +30,10 @@ func (o OrderByValue) Compare(a, b Entry) int { return bytes.Compare(a.Value, b.Value) } +func (OrderByValue) String() string { + return "VALUE" +} + // OrderByValueDescending is used to signal to datastores they // should apply internal orderings. type OrderByValueDescending struct{} @@ -34,6 +42,10 @@ func (o OrderByValueDescending) Compare(a, b Entry) int { return -bytes.Compare(a.Value, b.Value) } +func (OrderByValueDescending) String() string { + return "desc(VALUE)" +} + // OrderByKey type OrderByKey struct{} @@ -41,6 +53,10 @@ func (o OrderByKey) Compare(a, b Entry) int { return strings.Compare(a.Key, b.Key) } +func (OrderByKey) String() string { + return "KEY" +} + // OrderByKeyDescending type OrderByKeyDescending struct{} @@ -48,6 +64,10 @@ func (o OrderByKeyDescending) Compare(a, b Entry) int { return -strings.Compare(a.Key, b.Key) } +func (OrderByKeyDescending) String() string { + return "desc(KEY)" +} + // Less returns true if a comes before b with the requested orderings. func Less(orders []Order, a, b Entry) bool { for _, cmp := range orders { diff --git a/query/query.go b/query/query.go index 2540dfc67c3cdfca82f9afb8d7f82f9e2e59aa10..0b32ec9967302819f7090f0ea4e2a7ec48266bbf 100644 --- a/query/query.go +++ b/query/query.go @@ -1,6 +1,7 @@ package query import ( + "fmt" "time" goprocess "github.com/jbenet/goprocess" @@ -67,6 +68,48 @@ type Query struct { ReturnExpirations bool // return expirations (see TTLDatastore) } +func (q Query) String() string { + s := "SELECT keys" + if !q.KeysOnly { + s += ",vals" + } + if q.ReturnExpirations { + s += ",exps" + } + + s += " " + + if q.Prefix != "" { + s += fmt.Sprintf("FROM %q ", q.Prefix) + } + + if len(q.Filters) > 0 { + s += fmt.Sprintf("FILTER [%s", q.Filters[0]) + for _, f := range q.Filters[1:] { + s += fmt.Sprintf(", %s", f) + } + s += "] " + } + + if len(q.Orders) > 0 { + s += fmt.Sprintf("ORDER [%s", q.Orders[0]) + for _, f := range q.Orders[1:] { + s += fmt.Sprintf(", %s", f) + } + s += "] " + } + + if q.Offset > 0 { + s += fmt.Sprintf("OFFSET %d ", q.Offset) + } + + if q.Limit > 0 { + s += fmt.Sprintf("LIMIT %d ", q.Limit) + } + // Will always end with a space, strip it. + return s[:len(s)-1] +} + // Entry is a query result entry. type Entry struct { Key string // cant be ds.Key because circular imports ...!!! diff --git a/query/query_test.go b/query/query_test.go index 228c69a4d391106dd6e3caa87e9121edda2f3928..7f08cb05d06c584fd6132cc57445850450b23971 100644 --- a/query/query_test.go +++ b/query/query_test.go @@ -248,3 +248,65 @@ func getKeysViaChan(rs Results) []string { } return ret } + +func TestStringer(t *testing.T) { + q := Query{} + + expected := `SELECT keys,vals` + actual := q.String() + if actual != expected { + t.Fatalf("expected\n\t%s\ngot\n\t%s", expected, actual) + } + + q.Offset = 10 + q.Limit = 10 + expected = `SELECT keys,vals OFFSET 10 LIMIT 10` + actual = q.String() + if actual != expected { + t.Fatalf("expected\n\t%s\ngot\n\t%s", expected, actual) + } + + q.Orders = []Order{OrderByValue{}, OrderByKey{}} + expected = `SELECT keys,vals ORDER [VALUE, KEY] OFFSET 10 LIMIT 10` + actual = q.String() + if actual != expected { + t.Fatalf("expected\n\t%s\ngot\n\t%s", expected, actual) + } + + q.Filters = []Filter{ + FilterKeyCompare{Op: GreaterThan, Key: "/foo/bar"}, + FilterKeyCompare{Op: LessThan, Key: "/foo/bar"}, + } + expected = `SELECT keys,vals FILTER [KEY > "/foo/bar", KEY < "/foo/bar"] ORDER [VALUE, KEY] OFFSET 10 LIMIT 10` + actual = q.String() + if actual != expected { + t.Fatalf("expected\n\t%s\ngot\n\t%s", expected, actual) + } + + q.Prefix = "/foo" + expected = `SELECT keys,vals FROM "/foo" FILTER [KEY > "/foo/bar", KEY < "/foo/bar"] ORDER [VALUE, KEY] OFFSET 10 LIMIT 10` + actual = q.String() + if actual != expected { + t.Fatalf("expected\n\t%s\ngot\n\t%s", expected, actual) + } + + q.ReturnExpirations = true + expected = `SELECT keys,vals,exps FROM "/foo" FILTER [KEY > "/foo/bar", KEY < "/foo/bar"] ORDER [VALUE, KEY] OFFSET 10 LIMIT 10` + actual = q.String() + if actual != expected { + t.Fatalf("expected\n\t%s\ngot\n\t%s", expected, actual) + } + + q.KeysOnly = true + expected = `SELECT keys,exps FROM "/foo" FILTER [KEY > "/foo/bar", KEY < "/foo/bar"] ORDER [VALUE, KEY] OFFSET 10 LIMIT 10` + actual = q.String() + if actual != expected { + t.Fatalf("expected\n\t%s\ngot\n\t%s", expected, actual) + } + q.ReturnExpirations = false + expected = `SELECT keys FROM "/foo" FILTER [KEY > "/foo/bar", KEY < "/foo/bar"] ORDER [VALUE, KEY] OFFSET 10 LIMIT 10` + actual = q.String() + if actual != expected { + t.Fatalf("expected\n\t%s\ngot\n\t%s", expected, actual) + } +} diff --git a/test/basic_tests.go b/test/basic_tests.go index 08c8c74f34743b930c3c68586cef2e0d024b7a5f..862a634f238a4be96160f1ec4409f77bee9c6e4a 100644 --- a/test/basic_tests.go +++ b/test/basic_tests.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "math/rand" + "reflect" "strings" "testing" @@ -123,6 +124,30 @@ func SubtestNotFounds(t *testing.T, ds dstore.Datastore) { } } +func SubtestLimit(t *testing.T, ds dstore.Datastore) { + test := func(offset, limit int) { + t.Run(fmt.Sprintf("Slice/%d/%d", offset, limit), func(t *testing.T) { + subtestQuery(t, ds, dsq.Query{ + Orders: []dsq.Order{dsq.OrderByKey{}}, + Offset: offset, + Limit: limit, + KeysOnly: true, + }, 100) + }) + } + test(0, 10) + test(0, 0) + test(10, 0) + test(10, 10) + test(10, 20) + test(50, 20) + test(99, 20) + test(200, 20) + test(200, 0) + test(99, 0) + test(95, 0) +} + func SubtestOrder(t *testing.T, ds dstore.Datastore) { test := func(orders ...dsq.Order) { var types []string @@ -133,19 +158,7 @@ func SubtestOrder(t *testing.T, ds dstore.Datastore) { t.Run(name, func(t *testing.T) { subtestQuery(t, ds, dsq.Query{ Orders: orders, - }, func(t *testing.T, input, output []dsq.Entry) { - if len(input) != len(output) { - t.Fatal("got wrong number of keys back") - } - - dsq.Sort(orders, input) - - for i, e := range output { - if input[i].Key != e.Key { - t.Fatalf("in key output, got %s but expected %s", e.Key, input[i].Key) - } - } - }) + }, 100) }) } test(dsq.OrderByKey{}) @@ -160,20 +173,7 @@ func SubtestOrder(t *testing.T, ds dstore.Datastore) { } func SubtestManyKeysAndQuery(t *testing.T, ds dstore.Datastore) { - subtestQuery(t, ds, dsq.Query{KeysOnly: true}, func(t *testing.T, input, output []dsq.Entry) { - if len(input) != len(output) { - t.Fatal("got wrong number of keys back") - } - - dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, input) - dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, output) - - for i, e := range output { - if input[i].Key != e.Key { - t.Fatalf("in key output, got %s but expected %s", e.Key, input[i].Key) - } - } - }) + subtestQuery(t, ds, dsq.Query{KeysOnly: true}, 100) } // need a custom test filter to test the "fallback" filter case for unknown @@ -184,6 +184,79 @@ func (testFilter) Filter(e dsq.Entry) bool { return len(e.Key)%2 == 0 } +func SubtestCombinations(t *testing.T, ds dstore.Datastore) { + offsets := []int{ + 0, + 10, + 95, + 100, + } + limits := []int{ + 0, + 1, + 10, + 100, + } + filters := [][]dsq.Filter{ + {dsq.FilterKeyCompare{ + Op: dsq.Equal, + Key: "/0key0", + }}, + {dsq.FilterKeyCompare{ + Op: dsq.LessThan, + Key: "/2", + }}, + } + orders := [][]dsq.Order{ + {dsq.OrderByKey{}}, + {dsq.OrderByKeyDescending{}}, + {dsq.OrderByValue{}, dsq.OrderByKey{}}, + {dsq.OrderByFunction(func(a, b dsq.Entry) int { return bytes.Compare(a.Value, b.Value) })}, + } + lengths := []int{ + 0, + 1, + 100, + } + perms( + func(perm []int) { + q := dsq.Query{ + Offset: offsets[perm[0]], + Limit: limits[perm[1]], + Filters: filters[perm[2]], + Orders: orders[perm[3]], + } + length := lengths[perm[4]] + + t.Run(strings.ReplaceAll(fmt.Sprintf("%d/{%s}", length, q), " ", "ยท"), func(t *testing.T) { + subtestQuery(t, ds, q, length) + }) + }, + len(offsets), + len(limits), + len(filters), + len(orders), + len(lengths), + ) +} + +func perms(cb func([]int), ops ...int) { + current := make([]int, len(ops)) +outer: + for { + for i := range current { + if current[i] < (ops[i] - 1) { + current[i]++ + cb(current) + continue outer + } + current[i] = 0 + } + // out of permutations + return + } +} + func SubtestFilter(t *testing.T, ds dstore.Datastore) { test := func(filters ...dsq.Filter) { var types []string @@ -194,31 +267,7 @@ func SubtestFilter(t *testing.T, ds dstore.Datastore) { t.Run(name, func(t *testing.T) { subtestQuery(t, ds, dsq.Query{ Filters: filters, - }, func(t *testing.T, input, output []dsq.Entry) { - var exp []dsq.Entry - input: - for _, e := range input { - for _, f := range filters { - if !f.Filter(e) { - continue input - } - } - exp = append(exp, e) - } - - if len(exp) != len(output) { - t.Fatalf("got wrong number of keys back: expected %d, got %d", len(exp), len(output)) - } - - dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, exp) - dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, output) - - for i, e := range output { - if exp[i].Key != e.Key { - t.Fatalf("in key output, got %s but expected %s", e.Key, exp[i].Key) - } - } - }) + }, 100) }) } test(dsq.FilterKeyCompare{ @@ -258,9 +307,8 @@ func randValue() []byte { return value } -func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query, check func(t *testing.T, input, output []dsq.Entry)) { +func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query, count int) { var input []dsq.Entry - count := 100 for i := 0; i < count; i++ { s := fmt.Sprintf("%dkey%d", i, i) key := dstore.NewKey(s).String() @@ -297,14 +345,38 @@ func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query, check func(t * t.Fatal("calling query: ", err) } + if rq := resp.Query(); !reflect.DeepEqual(rq, q) { + t.Errorf("returned query\n %s\nexpected query\n %s", &rq, q) + } + t.Log("aggregating query results") - output, err := resp.Rest() + actual, err := resp.Rest() if err != nil { t.Fatal("query result error: ", err) } t.Log("verifying query output") - check(t, input, output) + expected, err := dsq.NaiveQueryApply(q, dsq.ResultsWithEntries(q, input)).Rest() + if err != nil { + t.Fatal("naive query error: ", err) + } + if len(actual) != len(expected) { + t.Fatalf("expected %d results, got %d", len(expected), len(actual)) + } + if len(q.Orders) == 0 { + dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, actual) + dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, expected) + } + for i := range actual { + if actual[i].Key != expected[i].Key { + t.Errorf("for result %d, expected key %q, got %q", i, expected[i].Key, actual[i].Key) + continue + } + if !q.KeysOnly && !bytes.Equal(actual[i].Value, expected[i].Value) { + t.Errorf("value mismatch for result %d (key=%q)", i, expected[i].Key) + } + + } t.Log("deleting all keys") for _, e := range input { diff --git a/test/suite.go b/test/suite.go index 3c565e590753d1826a49aafd1a6f2d32b08e6d1f..20d0f69d2220bd75ef2cf9291e5b71f74ee45527 100644 --- a/test/suite.go +++ b/test/suite.go @@ -13,7 +13,9 @@ import ( var BasicSubtests = []func(t *testing.T, ds dstore.Datastore){ SubtestBasicPutGet, SubtestNotFounds, + SubtestCombinations, SubtestOrder, + SubtestLimit, SubtestFilter, SubtestManyKeysAndQuery, }