From 990adf517df64235c85ffcb77e490f97fe2dfd6d Mon Sep 17 00:00:00 2001 From: Peter Ohler Date: Sat, 26 Oct 2024 18:02:40 -0400 Subject: [PATCH] Expr walk (#188) * Determine function to walk expr matches and allow modification * Add reflection to child, nth, and wildcard * Add descent and filter to expr.Walk --- CHANGELOG.md | 4 + jp/at.go | 9 ++ jp/bracket.go | 9 ++ jp/child.go | 28 ++++- jp/descent.go | 5 + jp/filter.go | 136 +++++++++++++++++++++++ jp/frag.go | 7 ++ jp/get.go | 46 ++++---- jp/has.go | 16 +-- jp/modify.go | 18 +-- jp/nth.go | 45 +++++++- jp/root.go | 9 ++ jp/script_test.go | 1 + jp/set.go | 26 ++--- jp/slice.go | 31 ++++++ jp/union.go | 16 ++- jp/walk.go | 10 ++ jp/walk_test.go | 275 ++++++++++++++++++++++++++++++++++++++++++++++ jp/wildcard.go | 177 +++++++++++++++++++++++++++++ notes | 1 - 20 files changed, 811 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 546f010..2ce30d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm The structure and content of this file follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.25.0] - 2024-10-26 +### Added +- The `Expr.Walk()` function was added. Similar to jp.Walk but walk expression matches. + ## [1.24.1] - 2024-09-15 ### Fixed - Fixed reflection map key matches when keys are string derivitives. diff --git a/jp/at.go b/jp/at.go index 6c5a6c0..a941388 100644 --- a/jp/at.go +++ b/jp/at.go @@ -18,3 +18,12 @@ func (f At) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { } return } + +// Walk continues with the next in rest. +func (f At) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } +} diff --git a/jp/bracket.go b/jp/bracket.go index 2737330..aca54ac 100644 --- a/jp/bracket.go +++ b/jp/bracket.go @@ -18,3 +18,12 @@ func (f Bracket) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { } return } + +// Walk continues with the next in rest. +func (f Bracket) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } +} diff --git a/jp/child.go b/jp/child.go index bab381b..6f9e9e8 100644 --- a/jp/child.go +++ b/jp/child.go @@ -79,10 +79,36 @@ func (f Child) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { case Keyed: v, has = td.ValueForKey(string(f)) default: - v, has = pp.reflectGetChild(td, string(f)) + v, has = reflectGetChild(td, string(f)) } if has { locs = locateNthChildHas(pp, f, v, rest, max) } return } + +// Walk follows the matching element in a map or map like element. +func (f Child) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + var ( + value any + has bool + ) + switch tv := nodes[len(nodes)-1].(type) { + case map[string]any: + value, has = tv[string(f)] + case gen.Object: + value, has = tv[string(f)] + case Keyed: + value, has = tv.ValueForKey(string(f)) + default: + value, has = reflectGetChild(tv, string(f)) + } + if has { + path = append(path, f) + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, append(nodes, value), cb) + } else { + cb(path, append(nodes, value)) + } + } +} diff --git a/jp/descent.go b/jp/descent.go index dcb3678..f476721 100644 --- a/jp/descent.go +++ b/jp/descent.go @@ -147,3 +147,8 @@ func (f Descent) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { } return } + +// Walk each element in the tree of elements. +func (f Descent) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + wildWalk(rest, path, nodes, cb, f) +} diff --git a/jp/filter.go b/jp/filter.go index e22ab61..fe3c75a 100644 --- a/jp/filter.go +++ b/jp/filter.go @@ -312,3 +312,139 @@ func (f Filter) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { } return } + +// Walk each element that matches the filter. +func (f *Filter) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + path = append(path, nil) + data := nodes[len(nodes)-1] + nodes = append(nodes, nil) + switch tv := data.(type) { + case []any: + for i, v := range tv { + if f.Match(v) { + path[len(path)-1] = Nth(i) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + } + } + case Indexed: + size := tv.Size() + for i := 0; i < size; i++ { + v := tv.ValueAtIndex(i) + if f.Match(v) { + path[len(path)-1] = Nth(i) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + } + } + case gen.Array: + for i, v := range tv { + if f.Match(v) { + path[len(path)-1] = Nth(i) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + } + } + case map[string]any: + if 0 < len(tv) { + keys := make([]string, 0, len(tv)) + for k := range tv { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if f.Match(tv[k]) { + path[len(path)-1] = Child(k) + nodes[len(nodes)-1] = tv[k] + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + } + } + } + case gen.Object: + if 0 < len(tv) { + keys := make([]string, 0, len(tv)) + for k := range tv { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if f.Match(tv[k]) { + path[len(path)-1] = Child(k) + nodes[len(nodes)-1] = tv[k] + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + } + } + } + case Keyed: + keys := tv.Keys() + sort.Strings(keys) + for _, key := range keys { + v, _ := tv.ValueForKey(key) + if f.Match(v) { + path[len(path)-1] = Child(key) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + } + } + default: + rv := reflect.ValueOf(tv) + switch rv.Kind() { + case reflect.Slice: + cnt := rv.Len() + for i := 0; i < cnt; i++ { + v := rv.Index(i).Interface() + if f.Match(v) { + path[len(path)-1] = Nth(i) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + } + } + case reflect.Map: + keys := rv.MapKeys() + sort.Slice(keys, func(i, j int) bool { + return strings.Compare(keys[i].String(), keys[j].String()) < 0 + }) + for _, k := range keys { + mv := rv.MapIndex(k) + v := mv.Interface() + if f.Match(v) { + path[len(path)-1] = Child(k.String()) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + } + } + } + } +} diff --git a/jp/frag.go b/jp/frag.go index dfdcc77..257dd0c 100644 --- a/jp/frag.go +++ b/jp/frag.go @@ -11,4 +11,11 @@ type Frag interface { Append(buf []byte, bracket, first bool) []byte locate(pp Expr, data any, rest Expr, max int) (locs []Expr) + + // Walk the matching elements in tail of nodes and call cb on the matches + // or follow on to the matching if not the last fragment in an + // expression. The rest argument is the rest of the expression after this + // fragment. The path is the normalized path up to this point. The nodes + // argument is the chain of data elements to the current location. + Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) } diff --git a/jp/get.go b/jp/get.go index f8b965d..6126482 100644 --- a/jp/get.go +++ b/jp/get.go @@ -70,7 +70,7 @@ func (x Expr) Get(data any) (results []any) { case Keyed: v, has = tv.ValueForKey(string(tf)) default: - v, has = x.reflectGetChild(tv, string(tf)) + v, has = reflectGetChild(tv, string(tf)) } if has { if int(fi) == len(x)-1 { // last one @@ -119,7 +119,7 @@ func (x Expr) Get(data any) (results []any) { has = true } default: - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } if has { if int(fi) == len(x)-1 { // last one @@ -262,7 +262,7 @@ func (x Expr) Get(data any) (results []any) { } } default: - got := x.reflectGetWild(tv) + got := reflectGetWild(tv) if int(fi) == len(x)-1 { // last one for i := len(got) - 1; 0 <= i; i-- { results = append(results, got[i]) @@ -433,7 +433,7 @@ func (x Expr) Get(data any) (results []any) { } } default: - got := x.reflectGetWild(tv) + got := reflectGetWild(tv) stack[len(stack)-1] = prev stack = append(stack, di|descentFlag) if int(fi) == len(x)-1 { // last one @@ -495,7 +495,7 @@ func (x Expr) Get(data any) (results []any) { case gen.Object: v, has = tv[tu] default: - v, has = x.reflectGetChild(tv, tu) + v, has = reflectGetChild(tv, tu) } case int64: i := int(tu) @@ -525,7 +525,7 @@ func (x Expr) Get(data any) (results []any) { has = true } default: - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } } if has { @@ -546,7 +546,7 @@ func (x Expr) Get(data any) (results []any) { case gen.Object: v, has = tv[tu] default: - v, has = x.reflectGetChild(tv, tu) + v, has = reflectGetChild(tv, tu) } case int64: i := int(tu) @@ -576,7 +576,7 @@ func (x Expr) Get(data any) (results []any) { has = true } default: - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } } if has { @@ -801,7 +801,7 @@ func (x Expr) Get(data any) (results []any) { } } default: - got := x.reflectGetSlice(tv, start, end, step) + got := reflectGetSlice(tv, start, end, step) if int(fi) == len(x)-1 { // last one for i := len(got) - 1; 0 <= i; i-- { results = append(results, got[i]) @@ -902,7 +902,7 @@ func (x Expr) FirstFound(data any) (any, bool) { case gen.Object: v, has = tv[string(tf)] default: - v, has = x.reflectGetChild(tv, string(tf)) + v, has = reflectGetChild(tv, string(tf)) } if has { if int(fi) == len(x)-1 { // last one @@ -950,7 +950,7 @@ func (x Expr) FirstFound(data any) (any, bool) { has = true } default: - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } if has { if int(fi) == len(x)-1 { // last one @@ -1093,7 +1093,7 @@ func (x Expr) FirstFound(data any) (any, bool) { } } default: - if v, has = x.reflectGetWildOne(tv); has { + if v, has = reflectGetWildOne(tv); has { if int(fi) == len(x)-1 { // last one return v, true } @@ -1253,7 +1253,7 @@ func (x Expr) FirstFound(data any) (any, bool) { } } default: - got := x.reflectGetWild(tv) + got := reflectGetWild(tv) stack[len(stack)-1] = prev stack = append(stack, di|descentFlag) if int(fi) == len(x)-1 { // last one @@ -1308,7 +1308,7 @@ func (x Expr) FirstFound(data any) (any, bool) { case gen.Object: v, has = tv[tu] default: - v, has = x.reflectGetChild(tv, tu) + v, has = reflectGetChild(tv, tu) } case int64: i := int(tu) @@ -1338,7 +1338,7 @@ func (x Expr) FirstFound(data any) (any, bool) { has = true } default: - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } } if has { @@ -1359,7 +1359,7 @@ func (x Expr) FirstFound(data any) (any, bool) { case gen.Object: v, has = tv[tu] default: - v, has = x.reflectGetChild(tv, tu) + v, has = reflectGetChild(tv, tu) } case int64: i := int(tu) @@ -1389,7 +1389,7 @@ func (x Expr) FirstFound(data any) (any, bool) { has = true } default: - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } } if has { @@ -1596,7 +1596,7 @@ func (x Expr) FirstFound(data any) (any, bool) { } } default: - if v, has = x.reflectGetNth(tv, start); has { + if v, has = reflectGetNth(tv, start); has { if int(fi) == len(x)-1 { // last one return v, true } @@ -1638,7 +1638,7 @@ func (x Expr) FirstFound(data any) (any, bool) { return nil, false } -func (x Expr) reflectGetChild(data any, key string) (v any, has bool) { +func reflectGetChild(data any, key string) (v any, has bool) { if !isNil(data) { rd := reflect.ValueOf(data) rt := rd.Type() @@ -1814,7 +1814,7 @@ func reflectGetStructFieldByNameOrJsonTag(structValue reflect.Value, key string) return } -func (x Expr) reflectGetNth(data any, i int) (v any, has bool) { +func reflectGetNth(data any, i int) (v any, has bool) { if !isNil(data) { rd := reflect.ValueOf(data) rt := rd.Type() @@ -1836,7 +1836,7 @@ func (x Expr) reflectGetNth(data any, i int) (v any, has bool) { return } -func (x Expr) reflectGetWild(data any) (va []any) { +func reflectGetWild(data any) (va []any) { if !isNil(data) { rd := reflect.ValueOf(data) rt := rd.Type() @@ -1865,7 +1865,7 @@ func (x Expr) reflectGetWild(data any) (va []any) { return } -func (x Expr) reflectGetWildOne(data any) (any, bool) { +func reflectGetWildOne(data any) (any, bool) { if !isNil(data) { rd := reflect.ValueOf(data) rt := rd.Type() @@ -1894,7 +1894,7 @@ func (x Expr) reflectGetWildOne(data any) (any, bool) { return nil, false } -func (x Expr) reflectGetSlice(data any, start, end, step int) (va []any) { +func reflectGetSlice(data any, start, end, step int) (va []any) { if !isNil(data) { rd := reflect.ValueOf(data) rt := rd.Type() diff --git a/jp/has.go b/jp/has.go index 809ab85..3f4aa76 100644 --- a/jp/has.go +++ b/jp/has.go @@ -52,7 +52,7 @@ func (x Expr) Has(data any) bool { v, has = tv[string(tf)] default: if !isNil(tv) { - v, has = x.reflectGetChild(tv, string(tf)) + v, has = reflectGetChild(tv, string(tf)) } } if has { @@ -103,7 +103,7 @@ func (x Expr) Has(data any) bool { } default: if !isNil(tv) { - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } } if has { @@ -251,7 +251,7 @@ func (x Expr) Has(data any) bool { } } default: - if v, has = x.reflectGetWildOne(tv); has { + if v, has = reflectGetWildOne(tv); has { if int(fi) == len(x)-1 { // last one return true } @@ -446,7 +446,7 @@ func (x Expr) Has(data any) bool { case gen.Object: v, has = tv[tu] default: - v, has = x.reflectGetChild(tv, tu) + v, has = reflectGetChild(tv, tu) } case int64: i := int(tu) @@ -476,7 +476,7 @@ func (x Expr) Has(data any) bool { has = true } default: - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } } if has { @@ -497,7 +497,7 @@ func (x Expr) Has(data any) bool { case gen.Object: v, has = tv[tu] default: - v, has = x.reflectGetChild(tv, tu) + v, has = reflectGetChild(tv, tu) } case int64: i := int(tu) @@ -527,7 +527,7 @@ func (x Expr) Has(data any) bool { has = true } default: - v, has = x.reflectGetNth(tv, i) + v, has = reflectGetNth(tv, i) } } if has { @@ -734,7 +734,7 @@ func (x Expr) Has(data any) bool { } } default: - if v, has = x.reflectGetNth(tv, start); has { + if v, has = reflectGetNth(tv, start); has { if int(fi) == len(x)-1 { // last one return true } diff --git a/jp/modify.go b/jp/modify.go index e80ea2b..a55703b 100644 --- a/jp/modify.go +++ b/jp/modify.go @@ -165,10 +165,10 @@ done: } } default: - if v, has = wx.reflectGetChild(tv, key); has { + if v, has = reflectGetChild(tv, key); has { if int(fi) == len(wx)-1 { // last one if nv, changed := modifier(v); changed { - wx.reflectSetChild(tv, key, nv) + reflectSetChild(tv, key, nv) if one && changed { break done } @@ -237,10 +237,10 @@ done: } default: var has bool - if v, has = wx.reflectGetNth(tv, i); has { + if v, has = reflectGetNth(tv, i); has { if int(fi) == len(wx)-1 { // last one if nv, changed := modifier(v); changed { - wx.reflectSetNth(tv, i, nv) + reflectSetNth(tv, i, nv) if one && changed { break done } @@ -384,7 +384,7 @@ done: } } } else { - for _, v := range wx.reflectGetWild(tv) { + for _, v := range reflectGetWild(tv) { stack = stackAddValue(stack, v) } } @@ -439,10 +439,10 @@ done: } default: var has bool - if v, has = wx.reflectGetChild(tv, tu); has { + if v, has = reflectGetChild(tv, tu); has { if int(fi) == len(wx)-1 { // last one if nv, changed := modifier(v); changed { - wx.reflectSetChild(tv, tu, nv) + reflectSetChild(tv, tu, nv) if one && changed { break done } @@ -530,7 +530,7 @@ done: } } else { var has bool - if v, has = wx.reflectGetNth(tv, i); has { + if v, has = reflectGetNth(tv, i); has { stack = stackAddValue(stack, v) } } @@ -736,7 +736,7 @@ done: } } } else { - for _, v = range wx.reflectGetSlice(tv, start, end, step) { + for _, v = range reflectGetSlice(tv, start, end, step) { stack = stackAddValue(stack, v) } } diff --git a/jp/nth.go b/jp/nth.go index 3d4d29c..d8a0b70 100644 --- a/jp/nth.go +++ b/jp/nth.go @@ -127,10 +127,53 @@ func (f Nth) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { has = true } default: - v, has = pp.reflectGetNth(td, i) + v, has = reflectGetNth(td, i) } if has { locs = locateNthChildHas(pp, Nth(i), v, rest, max) } return } + +// Walk follows the matching element in a slice or slice like element. +func (f Nth) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + var value any + index := int(f) + path = append(path, f) + switch tv := nodes[len(nodes)-1].(type) { + case []any: + if index < 0 { + index += len(tv) + } + if index < 0 || len(tv) <= index { + return + } + value = tv[index] + case gen.Array: + if index < 0 { + index += len(tv) + } + if index < 0 || len(tv) <= index { + return + } + value = tv[index] + case Indexed: + if index < 0 { + index += tv.Size() + } + if index < 0 || tv.Size() <= index { + return + } + value = tv.ValueAtIndex(index) + default: + var has bool + if value, has = reflectGetNth(tv, index); !has { + return + } + } + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, append(nodes, value), cb) + } else { + cb(path, append(nodes, value)) + } +} diff --git a/jp/root.go b/jp/root.go index eb8dac3..518e778 100644 --- a/jp/root.go +++ b/jp/root.go @@ -18,3 +18,12 @@ func (f Root) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { } return } + +// Walk continues with the next in rest. +func (f Root) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } +} diff --git a/jp/script_test.go b/jp/script_test.go index 30b8ea6..51b643d 100644 --- a/jp/script_test.go +++ b/jp/script_test.go @@ -96,6 +96,7 @@ func TestScriptParse(t *testing.T) { {src: "(@ has true)", expect: "(@ has true)"}, {src: "(@ exists true)", expect: "(@ exists true)"}, {src: "(@)", expect: "(@ exists true)"}, + {src: "(@.x)", expect: "(@.x exists true)"}, {src: "@", expect: "(@ exists true)"}, {src: "(@ =~ /abc/)", expect: "(@ ~= /abc/)"}, {src: "(@ ~= /a\\/c/)", expect: "(@ ~= /a\\/c/)"}, diff --git a/jp/set.go b/jp/set.go index c2b57ce..55370a9 100644 --- a/jp/set.go +++ b/jp/set.go @@ -244,11 +244,11 @@ func (x Expr) set(data, value any, fun string, one bool) error { default: if int(fi) == len(x)-1 { // last one if value != delFlag { - if x.reflectSetChild(tv, string(tf), value) && one { + if reflectSetChild(tv, string(tf), value) && one { return nil } } - } else if v, has = x.reflectGetChild(tv, string(tf)); has { + } else if v, has = reflectGetChild(tv, string(tf)); has { switch v.(type) { case nil, gen.Bool, gen.Int, gen.Float, gen.String, bool, string, float64, float32, int, uint, int8, int16, int32, int64, uint8, uint16, uint32, uint64: @@ -379,11 +379,11 @@ func (x Expr) set(data, value any, fun string, one bool) error { var has bool if int(fi) == len(x)-1 { // last one if value != delFlag { - if x.reflectSetNth(tv, i, value) && one { + if reflectSetNth(tv, i, value) && one { return nil } } - } else if v, has = x.reflectGetNth(tv, i); has { + } else if v, has = reflectGetNth(tv, i); has { switch v.(type) { case bool, string, float64, float32, int, uint, int8, int16, int32, int64, uint8, uint16, uint32, uint64, nil, gen.Bool, gen.Int, gen.Float, gen.String: @@ -600,11 +600,11 @@ func (x Expr) set(data, value any, fun string, one bool) error { if int(fi) != len(x)-1 { var va []any if one { - if vv, has := x.reflectGetWildOne(tv); has { + if vv, has := reflectGetWildOne(tv); has { va = []any{vv} } } else { - va = x.reflectGetWild(tv) + va = reflectGetWild(tv) } for _, v = range va { switch v.(type) { @@ -828,12 +828,12 @@ func (x Expr) set(data, value any, fun string, one bool) error { var has bool if int(fi) == len(x)-1 { // last one if value != delFlag { - if x.reflectSetChild(tv, tu, value) && one { + if reflectSetChild(tv, tu, value) && one { return nil } } - } else if v, has = x.reflectGetChild(tv, tu); has { + } else if v, has = reflectGetChild(tv, tu); has { switch v.(type) { case nil, gen.Bool, gen.Int, gen.Float, gen.String, bool, string, float64, float32, int, uint, int8, int16, int32, int64, uint8, uint16, uint32, uint64: @@ -947,11 +947,11 @@ func (x Expr) set(data, value any, fun string, one bool) error { var has bool if int(fi) == len(x)-1 { // last one if value != delFlag { - if x.reflectSetNth(tv, i, value) && one { + if reflectSetNth(tv, i, value) && one { return nil } } - } else if v, has = x.reflectGetNth(tv, i); has { + } else if v, has = reflectGetNth(tv, i); has { switch v.(type) { case nil, gen.Bool, gen.Int, gen.Float, gen.String, bool, string, float64, float32, int, uint, int8, int16, int32, int64, uint8, uint16, uint32, uint64: @@ -1125,7 +1125,7 @@ func (x Expr) set(data, value any, fun string, one bool) error { } default: if int(fi) != len(x)-1 { - for _, v = range x.reflectGetSlice(tv, start, end, step) { + for _, v = range reflectGetSlice(tv, start, end, step) { switch v.(type) { case nil, gen.Bool, gen.Int, gen.Float, gen.String, bool, string, float64, float32, int, uint, int8, int16, int32, int64, uint8, uint16, uint32, uint64: @@ -1163,7 +1163,7 @@ func (x Expr) set(data, value any, fun string, one bool) error { return nil } -func (x Expr) reflectSetChild(data any, key string, v any) bool { +func reflectSetChild(data any, key string, v any) bool { if !isNil(data) { rd := reflect.ValueOf(data) rt := rd.Type() @@ -1199,7 +1199,7 @@ func (x Expr) reflectSetChild(data any, key string, v any) bool { return false } -func (x Expr) reflectSetNth(data any, i int, v any) bool { +func reflectSetNth(data any, i int, v any) bool { if !isNil(data) { rd := reflect.ValueOf(data) rt := rd.Type() diff --git a/jp/slice.go b/jp/slice.go index ce5b9d0..62f9657 100644 --- a/jp/slice.go +++ b/jp/slice.go @@ -593,3 +593,34 @@ func (f Slice) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { } return } + +// Walk each element in a slice as defined by the Slice fragment. +func (f Slice) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + var max int + switch tn := nodes[len(nodes)-1].(type) { + case []any: + max = len(tn) + case gen.Array: + max = len(tn) + case Indexed: + max = tn.Size() + default: + rv := reflect.ValueOf(tn) + if rv.Kind() == reflect.Slice { + max = rv.Len() + } + } + start, end, step := f.startEndStep(max) + if step == 0 { + return + } + if 0 < step { + for i := start; i < end; i += step { + Nth(i).Walk(rest, path, nodes, cb) + } + } else { + for i := start; end < i; i += step { + Nth(i).Walk(rest, path, nodes, cb) + } + } +} diff --git a/jp/union.go b/jp/union.go index e2df6e7..df95025 100644 --- a/jp/union.go +++ b/jp/union.go @@ -313,7 +313,7 @@ func (f Union) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { case gen.Object: v, has = td[tu] default: - v, has = pp.reflectGetChild(td, tu) + v, has = reflectGetChild(td, tu) } lf = Child(tu) case int64: @@ -344,7 +344,7 @@ func (f Union) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { has = true } default: - v, has = pp.reflectGetNth(td, i) + v, has = reflectGetNth(td, i) } lf = Nth(i) } @@ -361,3 +361,15 @@ func (f Union) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { } return } + +// Walk each element in a union. +func (f Union) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + for _, u := range f { + switch tu := u.(type) { + case int64: + Nth(tu).Walk(rest, path, nodes, cb) + case string: + Child(tu).Walk(rest, path, nodes, cb) + } + } +} diff --git a/jp/walk.go b/jp/walk.go index f73daf7..ed940bd 100644 --- a/jp/walk.go +++ b/jp/walk.go @@ -71,3 +71,13 @@ top: cb(path, data) } } + +// Walk the matching elements in the data and call cb on the matches. The path +// passed to the cb function is the normalized path to the current location +// while the nodes are the chain of elements up to and including the current +// element. +func (x Expr) Walk(data any, cb func(path Expr, nodes []any)) { + if 0 < len(x) { + x[0].Walk(x[1:], Expr{}, []any{data}, cb) + } +} diff --git a/jp/walk_test.go b/jp/walk_test.go index 0aa89b0..469237e 100644 --- a/jp/walk_test.go +++ b/jp/walk_test.go @@ -3,9 +3,13 @@ package jp_test import ( + "bytes" "sort" + "strings" "testing" + "github.com/ohler55/ojg" + "github.com/ohler55/ojg/alt" "github.com/ohler55/ojg/gen" "github.com/ohler55/ojg/jp" "github.com/ohler55/ojg/pretty" @@ -15,6 +19,13 @@ import ( type simple int +type walkData struct { + path string + data string + xpath string + nodes string +} + func (s simple) Simplify() any { return map[string]any{"x": int64(s)} } @@ -42,3 +53,267 @@ func TestWalkNode(t *testing.T) { sort.Strings(paths) tt.Equal(t, `[$ $.a "$.a[0]" "$.a[1]" "$.a[2]" $.b]`, string(sen.Bytes(paths))) } + +var ( + walkTestData = []*walkData{ + {path: "a", data: "{a:1 b:{c:3}}", xpath: "a", nodes: "1"}, + {path: "b.c", data: "{a:1 b:{c:3}}", xpath: "b.c", nodes: "3"}, + {path: "[0]", data: "[1 [2 3]]", xpath: "[0]", nodes: "1"}, + {path: "[-1][-3]", data: "[1 [2 3]]", xpath: "", nodes: ""}, + {path: "[1][1]", data: "[1 [2 3]]", xpath: "[1][1]", nodes: "3"}, + {path: "*", data: "[1 [2 3]]", xpath: "[0] [1]", nodes: "1 [2 3]"}, + {path: "*.*", data: "[1 [2 3]]", xpath: "[1][0] [1][1]", nodes: "2 3"}, + {path: "*", data: "{a:1 b:{c:3}}", xpath: "a b", nodes: "1 {c:3}"}, + {path: "*.*", data: "{a:1 b:{c:3}}", xpath: "b.c", nodes: "3"}, + {path: "@", data: "{a:1}", xpath: "", nodes: "{a:1}"}, + {path: "@.a", data: "{a:1 b:{c:3}}", xpath: "a", nodes: "1"}, + {path: "$", data: "{a:1}", xpath: "", nodes: "{a:1}"}, + {path: "$.a", data: "{a:1 b:{c:3}}", xpath: "a", nodes: "1"}, + {path: "[1,'a']", data: "{a:1 b:{c:3}}", xpath: "a", nodes: "1"}, + {path: "['b','a']['c',2]", data: "{a:1 b:{c:3}}", xpath: "b.c", nodes: "3"}, + {path: "[0,'a']", data: "[1 [2 3]]", xpath: "[0]", nodes: "1"}, + {path: "[1,2][0,4]", data: "[1 [2 3]]", xpath: "[1][0]", nodes: "2"}, + {path: "[0:4:2]", data: "[1 2 3 4 5 6]", xpath: "[0] [2]", nodes: "1 3"}, + {path: "[0:4:0]", data: "[1 2 3 4 5 6]", xpath: "", nodes: ""}, + {path: "[4:0:-2]", data: "[1 2 3 4 5 6]", xpath: "[4] [2]", nodes: "5 3"}, + {path: "[?(@.x == 1)]", data: "[{x:0}{x:1}]", xpath: "[1]", nodes: "{x:1}"}, + {path: "[?(@.x == 1)].x", data: "[{x:0}{x:1}]", xpath: "[1].x", nodes: "1"}, + {path: "[?(@.x == 1)]", data: "{y:{x:0} z:{x:1}}", xpath: "z", nodes: "{x:1}"}, + {path: "[?(@.x == 1)].x", data: "{y:{x:0} z:{x:1}}", xpath: "z.x", nodes: "1"}, + {path: "..", data: "{a:1 b:{c:3}}", xpath: "a b b.c", nodes: "1 {c:3} 3"}, + {path: "..", data: "[1 [2 3]]", xpath: "[0] [1] [1][0] [1][1]", nodes: "1 [2 3] 2 3"}, + {path: "a.b.c", data: "{a:1 b:{c:3}}", xpath: "", nodes: ""}, + } +) + +func TestExprWalkAny(t *testing.T) { + testExprWalk(t, false) +} + +func TestExprWalkGen(t *testing.T) { + testExprWalk(t, true) +} + +func testExprWalk(t *testing.T, generic bool) { + opt := ojg.Options{Sort: true, Indent: 0} + for i, wd := range walkTestData { + x := jp.MustParseString(wd.path) + var ( + ps []string + ns []string + data any + ) + data = sen.MustParse([]byte(wd.data)) + if generic { + data = alt.Generify(data) + } + x.Walk(data, func(path jp.Expr, nodes []any) { + ps = append(ps, path.String()) + ns = append(ns, string(bytes.ReplaceAll(sen.Bytes(nodes[len(nodes)-1], &opt), []byte{'\n'}, []byte{}))) + }) + tt.Equal(t, wd.xpath, strings.Join(ps, " "), "%d: path mismatch for %s", i, wd.path) + tt.Equal(t, wd.nodes, strings.Join(ns, " "), "%d: nodes mismatch for %s", i, wd.path) + } +} + +func TestExprWalkBracket(t *testing.T) { + opt := ojg.Options{Sort: true, Indent: 0} + data := sen.MustParse([]byte("{a:1}")) + x := jp.B() + var ( + ps []byte + ns []byte + ) + x.Walk(data, func(path jp.Expr, nodes []any) { + ps = append(ps, path.String()...) + ns = append(ns, bytes.ReplaceAll(sen.Bytes(nodes[len(nodes)-1], &opt), []byte{'\n'}, []byte{})...) + }) + tt.Equal(t, "", string(ps)) + tt.Equal(t, "{a:1}", string(ns)) + + x = jp.B().C("a") + ps = ps[:0] + ns = ns[:0] + x.Walk(data, func(path jp.Expr, nodes []any) { + ps = append(ps, path.String()...) + ns = append(ns, bytes.ReplaceAll(sen.Bytes(nodes[len(nodes)-1], &opt), []byte{'\n'}, []byte{})...) + }) + tt.Equal(t, "a", string(ps)) + tt.Equal(t, "1", string(ns)) +} + +func TestExprWalkIndexed(t *testing.T) { + opt := ojg.Options{Sort: true, Indent: 0} + data := &indexed{ + ordered: ordered{ + entries: []*entry{ + {key: "a", value: 1}, + { + key: "b", + value: &indexed{ + ordered: ordered{ + entries: []*entry{ + {key: "b2", value: 2}, + {key: "b3", value: 3}, + }, + }, + }, + }, + }, + }, + } + for i, wd := range []*walkData{ + {path: "[0]", xpath: "[0]", nodes: "1"}, + {path: "[1][1]", xpath: "[1][1]", nodes: "3"}, + {path: "[-1][-3]", xpath: "", nodes: ""}, + {path: "*", xpath: "[0] [1]", nodes: "1 [{key:b2 value:2}{key:b3 value:3}]"}, + {path: "*.*", xpath: "[1][0] [1][1]", nodes: "2 3"}, + {path: "[0:4:2]", xpath: "[0]", nodes: "1"}, + {path: "[?(@ == 1)]", xpath: "[0]", nodes: "1"}, + {path: "[?(@[0] == 2)][1]", xpath: "[1][1]", nodes: "3"}, + {path: "..", xpath: "[0] [1] [1][0] [1][1]", nodes: "1 [{key:b2 value:2}{key:b3 value:3}] 2 3"}, + } { + x := jp.MustParseString(wd.path) + var ( + ps []string + ns []string + ) + x.Walk(data, func(path jp.Expr, nodes []any) { + ps = append(ps, path.String()) + ns = append(ns, string(bytes.ReplaceAll(sen.Bytes(nodes[len(nodes)-1], &opt), []byte{'\n'}, []byte{}))) + }) + tt.Equal(t, wd.xpath, strings.Join(ps, " "), "%d: path mismatch for %s", i, wd.path) + tt.Equal(t, wd.nodes, strings.Join(ns, " "), "%d: nodes mismatch for %s", i, wd.path) + } +} + +func TestExprWalkKeyed(t *testing.T) { + opt := ojg.Options{Sort: true, Indent: 0} + data := &keyed{ + ordered: ordered{ + entries: []*entry{ + {key: "a", value: 1}, + { + key: "b", + value: &keyed{ + ordered: ordered{ + entries: []*entry{ + {key: "c", value: 3}, + }, + }, + }, + }, + }, + }, + } + for i, wd := range []*walkData{ + {path: "a", xpath: "a", nodes: "1"}, + {path: "b.c", xpath: "b.c", nodes: "3"}, + {path: "*", xpath: "a b", nodes: "1 [{key:c value:3}]"}, + {path: "*.*", xpath: "b.c", nodes: "3"}, + {path: "[?(@ == 1)]", xpath: "a", nodes: "1"}, + {path: "[?(@.c == 3)].c", xpath: "b.c", nodes: "3"}, + {path: "..", xpath: "a b b.c", nodes: "1 [{key:c value:3}] 3"}, + } { + x := jp.MustParseString(wd.path) + var ( + ps []string + ns []string + ) + x.Walk(data, func(path jp.Expr, nodes []any) { + ps = append(ps, path.String()) + ns = append(ns, string(bytes.ReplaceAll(sen.Bytes(nodes[len(nodes)-1], &opt), []byte{'\n'}, []byte{}))) + }) + tt.Equal(t, wd.xpath, strings.Join(ps, " "), "%d: path mismatch for %s", i, wd.path) + tt.Equal(t, wd.nodes, strings.Join(ns, " "), "%d: nodes mismatch for %s", i, wd.path) + } +} + +func TestExprWalkSliceReflect(t *testing.T) { + opt := ojg.Options{Sort: true, Indent: 0} + type AA []any + data := AA{1, AA{2, 3}} + for i, wd := range []*walkData{ + {path: "[0]", xpath: "[0]", nodes: "1"}, + {path: "[1][1]", xpath: "[1][1]", nodes: "3"}, + {path: "[-1][-3]", xpath: "", nodes: ""}, + {path: "*", xpath: "[0] [1]", nodes: "1 [2 3]"}, + {path: "*.*", xpath: "[1][0] [1][1]", nodes: "2 3"}, + {path: "[0:4:2]", xpath: "[0]", nodes: "1"}, + {path: "[?(@ == 1)]", xpath: "[0]", nodes: "1"}, + {path: "[?(@[0] == 2)][1]", xpath: "[1][1]", nodes: "3"}, + {path: "..", xpath: "[0] [1] [1][0] [1][1]", nodes: "1 [2 3] 2 3"}, + } { + x := jp.MustParseString(wd.path) + var ( + ps []string + ns []string + ) + x.Walk(data, func(path jp.Expr, nodes []any) { + ps = append(ps, path.String()) + ns = append(ns, string(bytes.ReplaceAll(sen.Bytes(nodes[len(nodes)-1], &opt), []byte{'\n'}, []byte{}))) + }) + tt.Equal(t, wd.xpath, strings.Join(ps, " "), "%d: path mismatch for %s", i, wd.path) + tt.Equal(t, wd.nodes, strings.Join(ns, " "), "%d: nodes mismatch for %s", i, wd.path) + } +} + +func TestExprWalkStruct(t *testing.T) { + opt := ojg.Options{Sort: true, Indent: 0} + type B struct { + C int + } + type top struct { + A int + B B + } + data := &top{A: 1, B: B{C: 3}} + for i, wd := range []*walkData{ + {path: "a", xpath: "a", nodes: "1"}, + {path: "b.c", xpath: "b.c", nodes: "3"}, + {path: "*", xpath: "A B", nodes: "1 {c:3}"}, + {path: "*.*", xpath: "B.C", nodes: "3"}, + {path: "..", xpath: "A B B.C", nodes: "1 {c:3} 3"}, + } { + x := jp.MustParseString(wd.path) + var ( + ps []string + ns []string + ) + x.Walk(data, func(path jp.Expr, nodes []any) { + ps = append(ps, path.String()) + ns = append(ns, string(bytes.ReplaceAll(sen.Bytes(nodes[len(nodes)-1], &opt), []byte{'\n'}, []byte{}))) + }) + tt.Equal(t, wd.xpath, strings.Join(ps, " "), "%d: path mismatch for %s", i, wd.path) + tt.Equal(t, wd.nodes, strings.Join(ns, " "), "%d: nodes mismatch for %s", i, wd.path) + } +} + +func TestExprWalkMap(t *testing.T) { + opt := ojg.Options{Sort: true, Indent: 0} + type name string + type MM map[name]int + + data := map[name]any{"a": 1, "b": MM{"c": 3}} + for i, wd := range []*walkData{ + {path: "b", xpath: "b", nodes: "{c:3}"}, + {path: "b.c", xpath: "b.c", nodes: "3"}, + {path: "*", xpath: "a b", nodes: "1 {c:3}"}, + {path: "*.*", xpath: "b.c", nodes: "3"}, + {path: "[?(@.c == 3)]", xpath: "b", nodes: "{c:3}"}, + {path: "[?(@.c == 3)].c", xpath: "b.c", nodes: "3"}, + {path: "..", xpath: "a b b.c", nodes: "1 {c:3} 3"}, + } { + x := jp.MustParseString(wd.path) + var ( + ps []string + ns []string + ) + x.Walk(data, func(path jp.Expr, nodes []any) { + ps = append(ps, path.String()) + ns = append(ns, string(bytes.ReplaceAll(sen.Bytes(nodes[len(nodes)-1], &opt), []byte{'\n'}, []byte{}))) + }) + tt.Equal(t, wd.xpath, strings.Join(ps, " "), "%d: path mismatch for %s", i, wd.path) + tt.Equal(t, wd.nodes, strings.Join(ns, " "), "%d: nodes mismatch for %s", i, wd.path) + } +} diff --git a/jp/wildcard.go b/jp/wildcard.go index 13325ec..feef667 100644 --- a/jp/wildcard.go +++ b/jp/wildcard.go @@ -330,3 +330,180 @@ func (f Wildcard) locate(pp Expr, data any, rest Expr, max int) (locs []Expr) { } return } + +// Walk follows the all elements in a map or slice like element. +func (f Wildcard) Walk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any)) { + wildWalk(rest, path, nodes, cb, nil) +} + +func wildWalk(rest, path Expr, nodes []any, cb func(path Expr, nodes []any), f Frag) { + path = append(path, nil) + data := nodes[len(nodes)-1] + nodes = append(nodes, nil) + switch tv := data.(type) { + case []any: + for i, v := range tv { + path[len(path)-1] = Nth(i) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + case map[string]any: + if 0 < len(tv) { + keys := make([]string, 0, len(tv)) + for k := range tv { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := tv[k] + path[len(path)-1] = Child(k) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + } + case gen.Array: + for i, v := range tv { + path[len(path)-1] = Nth(i) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + case gen.Object: + if 0 < len(tv) { + keys := make([]string, 0, len(tv)) + for k := range tv { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := tv[k] + path[len(path)-1] = Child(k) + nodes[len(nodes)-1] = v + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + } + case Indexed: + for i := 0; i < tv.Size(); i++ { + path[len(path)-1] = Nth(i) + nodes[len(nodes)-1] = tv.ValueAtIndex(i) + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + case Keyed: + keys := tv.Keys() + sort.Strings(keys) + for _, k := range keys { + path[len(path)-1] = Child(k) + nodes[len(nodes)-1], _ = tv.ValueForKey(k) + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + case nil, bool, string, float64, float32, gen.Bool, gen.Float, gen.String, + int, uint, int8, int16, int32, int64, uint8, uint16, uint32, uint64, gen.Int: + // A bypass to avoid using reflection in the default case. + default: + if rt := reflect.TypeOf(tv); rt != nil { + rd := reflect.ValueOf(tv) + rwalk: + switch rt.Kind() { + case reflect.Ptr: + rt = rt.Elem() + rd = rd.Elem() + goto rwalk + case reflect.Struct: + cnt := rd.NumField() + for i := 0; i < cnt; i++ { + rv := rd.Field(i) + if rv.CanInterface() { + path[len(path)-1] = Child(rt.Field(i).Name) + nodes[len(nodes)-1] = rv.Interface() + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + } + case reflect.Slice, reflect.Array: + // Iterate in reverse order as that puts values on the stack in reverse. + for i := 0; i < rd.Len(); i++ { + rv := rd.Index(i) + if rv.CanInterface() { + path[len(path)-1] = Nth(i) + nodes[len(nodes)-1] = rv.Interface() + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + } + case reflect.Map: + keys := rd.MapKeys() + sort.Slice(keys, func(i, j int) bool { + return keys[i].String() < keys[j].String() + }) + for _, kv := range keys { + rv := rd.MapIndex(kv) + if rv.CanInterface() { + path[len(path)-1] = Child(kv.String()) + nodes[len(nodes)-1] = rv.Interface() + if 0 < len(rest) { + rest[0].Walk(rest[1:], path, nodes, cb) + } else { + cb(path, nodes) + } + if f != nil { + f.Walk(rest, path, nodes, cb) + } + } + } + } + } + } +} diff --git a/notes b/notes index 7787f2b..1fade71 100644 --- a/notes +++ b/notes @@ -1,5 +1,4 @@ - - @.foo without a comparison indicates existance - parse - add discover option to find JSON or SEN in a string or file