diff --git a/backend/balancer/db.go b/backend/balancer/db.go index 91268cb..04921f4 100644 --- a/backend/balancer/db.go +++ b/backend/balancer/db.go @@ -120,6 +120,8 @@ type tx struct { cfn CloseFunc } +func (tx *tx) Driver() string { return sql.DriverOf(tx.Tx) } + func (tx *tx) Commit() error { err := tx.Tx.Commit() diff --git a/backend/postgres/db.go b/backend/postgres/db.go index 967157c..41d5642 100644 --- a/backend/postgres/db.go +++ b/backend/postgres/db.go @@ -42,6 +42,7 @@ type tx struct { func (tx *tx) Commit() error { return wrapErr(tx.tx.Commit()) } func (tx *tx) Rollback() error { return wrapErr(tx.tx.Rollback()) } +func (tx *tx) Driver() string { return sql.DriverOf(tx.tx) } type queryer struct { q sql.Queryer diff --git a/backend/simple/db.go b/backend/simple/db.go index 97d7ba2..da7d07e 100644 --- a/backend/simple/db.go +++ b/backend/simple/db.go @@ -37,8 +37,12 @@ type tx struct { q *queryer tx *stdsql.Tx + + driver string } +func (tx *tx) Driver() string { return tx.driver } + func (tx *tx) Commit() error { select { case <-tx.ctx.Done(): @@ -160,9 +164,10 @@ func (d *db) BeginTx(ctx context.Context, opts sql.TxOptions) (sql.Tx, error) { } return &tx{ - ctx: ctx, - ch: make(chan struct{}, 1), - q: &queryer{t}, - tx: t, + ctx: ctx, + ch: make(chan struct{}, 1), + q: &queryer{t}, + tx: t, + driver: d.driver, }, nil } diff --git a/backend/sqlite3/db.go b/backend/sqlite3/db.go index 29d49dc..868f4c4 100644 --- a/backend/sqlite3/db.go +++ b/backend/sqlite3/db.go @@ -46,6 +46,7 @@ type tx struct { func (tx *tx) Commit() error { return wrapErr(tx.tx.Commit()) } func (tx *tx) Rollback() error { return wrapErr(tx.tx.Rollback()) } +func (tx *tx) Driver() string { return sql.DriverOf(tx.tx) } func (db *db) Driver() string { return db.db.Driver() } diff --git a/backend/static/db.go b/backend/static/db.go index 63641ef..f941989 100644 --- a/backend/static/db.go +++ b/backend/static/db.go @@ -57,6 +57,7 @@ type Tx struct { func (tx *Tx) Commit() error { return tx.CommitErr } func (tx *Tx) Rollback() error { return tx.RollbackErr } +func (tx *Tx) Driver() string { return "sqltest" } func (db *DB) Driver() string { return "sqltest" } diff --git a/db.go b/db.go index b6e4861..f9f13ca 100644 --- a/db.go +++ b/db.go @@ -40,6 +40,16 @@ type DB interface { Driver() string } +// DriverOf returns the driver of q, the DBs and the transactions they +// begin expose it. It returns an empty string when q does not. +func DriverOf(q Queryer) string { + if d, ok := q.(interface{ Driver() string }); ok { + return d.Driver() + } + + return "" +} + type TxOptions struct { Isolation IsolationLevel } diff --git a/integration/jsonb_test.go b/integration/jsonb_test.go new file mode 100644 index 0000000..b8c73c1 --- /dev/null +++ b/integration/jsonb_test.go @@ -0,0 +1,327 @@ +package integration + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/upfluence/sql" + "github.com/upfluence/sql/sqltest" + "github.com/upfluence/sql/x/migration" + "github.com/upfluence/sql/x/sqlbuilder" +) + +const jsonbAddressKey = "address" + +type jsonbBoundEntity struct { + Type string `json:"type"` + ID int64 `json:"id"` +} + +func jsonbMigrator(db sql.DB) migration.Migrator { + up := "CREATE TABLE jsonb_rows(id INTEGER PRIMARY KEY, tags TEXT, data TEXT, entities TEXT)" + + if db.Driver() == postgresDriver { + up = "CREATE TABLE jsonb_rows(id INTEGER PRIMARY KEY, tags JSONB, data JSONB, entities JSONB)" + } + + return migration.NewMigrator( + db, + staticSource{up: up, down: "DROP TABLE jsonb_rows"}, + ) +} + +func TestJSONBPredicateClause(t *testing.T) { + sqltest.NewTestCase( + sqltest.WithMigratorFunc(jsonbMigrator), + ).Run(t, func(t *testing.T, db sql.DB) { + ctx := context.Background() + + for _, row := range []struct { + id int64 + tags, data, entities any + }{ + { + id: 1, + tags: `["a", "b"]`, + data: `{"status": "active", "tier": 2, "address": {"country": "FR"}, "labels": ["x", "y"]}`, + entities: `[{"type": "campaign", "id": 12}, {"type": "contribution", "id": 14}]`, + }, + { + id: 2, + tags: `[1, 2.5]`, + data: `{"status": "inactive", "tier": 2.0, "address": {"country": "US"}}`, + entities: `[{"type": "campaign", "id": 13}]`, + }, + { + id: 3, + tags: `["1", true]`, + data: `{"status": "active", "tier": "2", "labels": "x"}`, + entities: `[{"type": "campaign", "id": 12, "extra": true}]`, + }, + { + id: 4, + tags: `{"a": "a"}`, + data: `["status"]`, + entities: `{"type": "campaign", "id": 12}`, + }, + {id: 5, tags: `[]`, data: `{}`, entities: `[]`}, + {id: 6}, + { + id: 7, + tags: `[["a"]]`, + data: `{"address": {"country": "FR", "city": "Paris"}, "labels": ["y", "x", "z"], "flags": true, "items": [{"sku": "x", "qty": 1}, {"sku": "y"}]}`, + entities: `[{"type": "contribution", "id": 12}, {"type": "campaign", "id": "12"}]`, + }, + } { + _, err := db.Exec( + ctx, + "INSERT INTO jsonb_rows(id, tags, data, entities) VALUES ($1, $2, $3, $4)", + row.id, + row.tags, + row.data, + row.entities, + ) + + require.NoError(t, err) + } + + var ( + tags = sqlbuilder.Column("tags") + data = sqlbuilder.Column("data") + entities = sqlbuilder.Column("entities") + + campaign12 = jsonbBoundEntity{Type: "campaign", ID: 12} + campaign13 = jsonbBoundEntity{Type: "campaign", ID: 13} + contribution14 = jsonbBoundEntity{Type: "contribution", ID: 14} + ) + + for _, tt := range []struct { + name string + haveClause sqlbuilder.PredicateClause + want []int64 + }{ + { + name: "contains any string", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny(tags, []string{"a"}), + want: []int64{1}, + }, + { + name: "contains any number", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny(tags, []int{1}), + want: []int64{2}, + }, + { + name: "contains any mixed", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny(tags, []any{2.5, "b"}), + want: []int64{1, 2}, + }, + { + name: "contains any of nothing", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny(tags, []string{}), + }, + { + name: "contains all", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{"b", "a"}), + want: []int64{1}, + }, + { + name: "contains all duplicated", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{"a", "a"}), + want: []int64{1}, + }, + { + name: "contains all numbers", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []float64{1, 2.5}), + want: []int64{2}, + }, + { + name: "contains all numeric string", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{"1"}), + want: []int64{3}, + }, + { + name: "contains all of nothing", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{}), + want: []int64{1, 2, 3, 5, 7}, + }, + { + name: "object contains string", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{"status": "active"}, + ), + want: []int64{1, 3}, + }, + { + name: "object contains number", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{"tier": 2}, + ), + want: []int64{1, 2}, + }, + { + name: "object contains nested object", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{jsonbAddressKey: map[string]any{"country": "FR"}}, + ), + want: []int64{1, 7}, + }, + { + name: "object contains empty object", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{jsonbAddressKey: map[string]any{}}, + ), + want: []int64{1, 2, 7}, + }, + { + name: "object contains array", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{"labels": []string{"x"}}, + ), + want: []int64{1, 7}, + }, + { + name: "object contains nothing", + haveClause: sqlbuilder.StaticJSONBObjectContains(data, map[string]any{}), + want: []int64{1, 2, 3, 5, 7}, + }, + { + name: "object contains many", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{ + "status": "active", + jsonbAddressKey: map[string]any{"country": "FR"}, + "labels": []string{"y", "x"}, + }, + ), + want: []int64{1}, + }, + { + name: "object contains array of objects", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{"items": []any{map[string]any{"sku": "x"}}}, + ), + want: []int64{7}, + }, + { + name: "contains all nested array", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []any{[]string{"a"}}), + want: []int64{7}, + }, + { + name: "contains any object", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny( + entities, + []jsonbBoundEntity{campaign12}, + ), + want: []int64{1, 3}, + }, + { + name: "contains any objects", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny( + entities, + []jsonbBoundEntity{ + campaign13, + contribution14, + }, + ), + want: []int64{1, 2}, + }, + { + name: "contains all objects", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll( + entities, + []jsonbBoundEntity{ + campaign12, + contribution14, + }, + ), + want: []int64{1}, + }, + { + name: "contains all object", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll( + entities, + []jsonbBoundEntity{campaign12}, + ), + want: []int64{1, 3}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := selectJSONBRowIDs(ctx, db, tt.haveClause) + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + + t.Run("in a transaction", func(t *testing.T) { + var got []int64 + + err := sql.ExecuteTx( + ctx, + db, + sql.TxOptions{}, + func(q sql.Queryer) error { + var err error + + got, err = selectJSONBRowIDs( + ctx, + q, + sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{"a"}), + ) + + return err + }, + ) + + require.NoError(t, err) + assert.Equal(t, []int64{1}, got) + }) + }) +} + +func selectJSONBRowIDs(ctx context.Context, q sql.Queryer, pc sqlbuilder.PredicateClause) ([]int64, error) { + qb := sqlbuilder.QueryBuilder{Queryer: q} + + cur, err := qb.PrepareSelect( + sqlbuilder.SelectStatement{ + Table: "jsonb_rows", + SelectClauses: []sqlbuilder.Marker{sqlbuilder.Column("id")}, + WhereClause: pc, + OrderByClauses: []sqlbuilder.OrderByClause{{Field: sqlbuilder.Column("id")}}, + }, + ).Query(ctx, nil) + + if err != nil { + return nil, err + } + + var ids []int64 + + err = sqlbuilder.ScrollCursor( + cur, + func(sc sqlbuilder.Scanner) error { + var id int64 + + if err := sc.Scan(map[string]any{"id": &id}); err != nil { + return err + } + + ids = append(ids, id) + + return nil + }, + ) + + return ids, err +} diff --git a/middleware/logger/middleware.go b/middleware/logger/middleware.go index a669f68..7e2edd1 100644 --- a/middleware/logger/middleware.go +++ b/middleware/logger/middleware.go @@ -101,6 +101,8 @@ type tx struct { tx sql.Tx } +func (t *tx) Driver() string { return sql.DriverOf(t.tx) } + func (t *tx) Commit() error { var t0 = time.Now() diff --git a/x/sqlbuilder/delete_statement.go b/x/sqlbuilder/delete_statement.go index 48cb3c5..c069751 100644 --- a/x/sqlbuilder/delete_statement.go +++ b/x/sqlbuilder/delete_statement.go @@ -15,8 +15,8 @@ func (ds DeleteStatement) Clone() DeleteStatement { } } -func (ds DeleteStatement) buildQuery(vs map[string]any) (string, []any, error) { - var qw queryWriter +func (ds DeleteStatement) buildQuery(driver string, vs map[string]any) (string, []any, error) { + qw := queryWriter{driver: driver} fmt.Fprintf(&qw, "DELETE FROM %s WHERE ", ds.Table) diff --git a/x/sqlbuilder/delete_statement_test.go b/x/sqlbuilder/delete_statement_test.go index af9ae42..0de6161 100644 --- a/x/sqlbuilder/delete_statement_test.go +++ b/x/sqlbuilder/delete_statement_test.go @@ -34,7 +34,7 @@ func TestDeleteQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, err := tt.ds.Clone().buildQuery(tt.vs) + stmt, args, err := tt.ds.Clone().buildQuery("", tt.vs) assert.Equal(t, tt.stmt, stmt) assert.Equal(t, tt.args, args) diff --git a/x/sqlbuilder/execer.go b/x/sqlbuilder/execer.go index 1d7d2ee..4e8e5a3 100644 --- a/x/sqlbuilder/execer.go +++ b/x/sqlbuilder/execer.go @@ -16,7 +16,7 @@ type execer struct { } func (e execer) Exec(ctx context.Context, qvs map[string]any) (sql.Result, error) { - stmt, vs, err := e.stmt.buildQuery(qvs) + stmt, vs, err := e.stmt.buildQuery(e.qb.driver(), qvs) if err != nil { return nil, err diff --git a/x/sqlbuilder/insert_statement.go b/x/sqlbuilder/insert_statement.go index d3b217d..189cb0c 100644 --- a/x/sqlbuilder/insert_statement.go +++ b/x/sqlbuilder/insert_statement.go @@ -156,12 +156,12 @@ func (is InsertStatement) returnings() []*sql.Returning { return res } -func (is InsertStatement) buildQuery(qvs map[string]any) (string, []any, error) { - return is.buildQueries([]map[string]any{qvs}, qvs) +func (is InsertStatement) buildQuery(driver string, qvs map[string]any) (string, []any, error) { + return is.buildQueries(driver, []map[string]any{qvs}, qvs) } -func (is InsertStatement) buildQueries(vvs []map[string]any, qvs map[string]any) (string, []any, error) { - var qw queryWriter +func (is InsertStatement) buildQueries(driver string, vvs []map[string]any, qvs map[string]any) (string, []any, error) { + qw := queryWriter{driver: driver} if len(is.Fields) == 0 { return "", nil, errNoMarkers diff --git a/x/sqlbuilder/insert_statement_test.go b/x/sqlbuilder/insert_statement_test.go index 55f39f2..a4c02e4 100644 --- a/x/sqlbuilder/insert_statement_test.go +++ b/x/sqlbuilder/insert_statement_test.go @@ -176,7 +176,7 @@ func TestInsertQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, err := tt.is.Clone().buildQuery(tt.vs) + stmt, args, err := tt.is.Clone().buildQuery("", tt.vs) assert.Equal(t, tt.err, err) assert.Equal(t, tt.stmt, stmt) @@ -237,7 +237,7 @@ func TestInsertMultiQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, err := tt.haveStmt.buildQueries(tt.haveVVs, nil) + stmt, args, err := tt.haveStmt.buildQueries("", tt.haveVVs, nil) assert.Equal(t, tt.wantErr, err) assert.Equal(t, tt.wantStmt, stmt) diff --git a/x/sqlbuilder/jsonb_predicate_clause.go b/x/sqlbuilder/jsonb_predicate_clause.go new file mode 100644 index 0000000..29e7016 --- /dev/null +++ b/x/sqlbuilder/jsonb_predicate_clause.go @@ -0,0 +1,489 @@ +package sqlbuilder + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "slices" + "strings" + + "github.com/upfluence/errors" +) + +const ( + sqlite3Driver = "sqlite3" + + // sqliteJSONBElementMatch matches the elements of a JSON array iterated + // as jsonb_elem with the ones of the array of wanted values iterated as + // jsonb_want. Values of different storage classes are never equal in + // SQLite, strings never match numbers, and 1 matches 1.0 as it does in + // PostgreSQL. + sqliteJSONBElementMatch = "jsonb_elem.type IN ('integer', 'real', 'text') " + + "AND jsonb_elem.value = jsonb_want.value" +) + +var ( + errInvalidJSONBValue = errors.New("JSONB predicates do not support booleans and nulls") + errInvalidJSONBKey = errors.New("SQLite does not support object keys holding a double quote") +) + +// JSONBArrayContainsAny matches the rows whose column, a JSONB array, holds +// at least one of the elements of the slice bound to the marker. An object +// or array element matches the elements containing it, as the @> operator +// of PostgreSQL does. The elements can not hold booleans and nulls. +func JSONBArrayContainsAny(m Marker) PredicateClause { + return &basicClause{m: m, fn: writeJSONBArrayContainsAny} +} + +// JSONBArrayContainsAll matches the rows whose column, a JSONB array, holds +// every element of the slice bound to the marker. An object or array element +// matches the elements containing it, as the @> operator of PostgreSQL does. +// The elements can not hold booleans and nulls. +func JSONBArrayContainsAll(m Marker) PredicateClause { + return &basicClause{m: m, fn: writeJSONBArrayContainsAll} +} + +// JSONBObjectContains matches the rows whose column, a JSONB object, +// contains the object bound to the marker, as the @> operator of +// PostgreSQL does. The object can not hold booleans and nulls. +func JSONBObjectContains(m Marker) PredicateClause { + return &basicClause{m: m, fn: writeJSONBObjectContains} +} + +func StaticJSONBArrayContainsAny(m Marker, v any) PredicateClause { + return Static(JSONBArrayContainsAny(m), map[string]any{m.Binding(): v}) +} + +func StaticJSONBArrayContainsAll(m Marker, v any) PredicateClause { + return Static(JSONBArrayContainsAll(m), map[string]any{m.Binding(): v}) +} + +func StaticJSONBObjectContains(m Marker, v any) PredicateClause { + return Static(JSONBObjectContains(m), map[string]any{m.Binding(): v}) +} + +func writeJSONBArrayContainsAny(w QueryWriter, vv any, k string) error { + es, err := decodeJSONBArray(vv) + + if err != nil { + return err + } + + if len(es) == 0 { + _, err := io.WriteString(w, "1=0") + + return errors.WithStack(err) + } + + if w.Driver() == sqlite3Driver { + sc := sqliteContains{w: w} + + cs, err := sc.containsAny(k, es) + + if err != nil { + return err + } + + _, err = fmt.Fprintf(w, "(json_type(%s) = 'array' AND %s)", k, cs) + + return errors.WithStack(err) + } + + // A disjunction of containments, unlike the ?| operator, matches + // numbers and objects and can use both the jsonb_ops and jsonb_path_ops + // GIN indexes. + cs := make([]string, len(es)) + + for i, e := range es { + v, err := encodeJSONB([]any{e}) + + if err != nil { + return err + } + + cs[i] = fmt.Sprintf("%s @> %s::jsonb", k, w.RedeemVariable(v)) + } + + _, err = fmt.Fprintf(w, "(%s)", strings.Join(cs, " OR ")) + + return errors.WithStack(err) +} + +func writeJSONBArrayContainsAll(w QueryWriter, vv any, k string) error { + es, err := decodeJSONBArray(vv) + + if err != nil { + return err + } + + return writeJSONBContains(w, k, es) +} + +func writeJSONBObjectContains(w QueryWriter, vv any, k string) error { + d, err := decodeJSONB(vv) + + if err != nil { + return err + } + + if _, ok := d.(map[string]any); !ok { + return errors.Wrapf(errInvalidJSONBValue, "%T is not an object", vv) + } + + return writeJSONBContains(w, k, d) +} + +func writeJSONBContains(w QueryWriter, k string, want any) error { + if w.Driver() != sqlite3Driver { + v, err := encodeJSONB(want) + + if err != nil { + return err + } + + _, err = fmt.Fprintf(w, "%s @> %s::jsonb", k, w.RedeemVariable(v)) + + return errors.WithStack(err) + } + + sc := sqliteContains{w: w} + + cs, err := sc.contains(k, "", want) + + if err != nil { + return err + } + + _, err = fmt.Fprintf(w, "(%s)", strings.Join(cs, " AND ")) + + return errors.WithStack(err) +} + +// sqliteContains writes the SQLite conditions matching a JSON document +// containing a wanted value, as the @> operator of PostgreSQL does. +type sqliteContains struct { + w QueryWriter + + // items is the number of aliases given to the iterations of the arrays + // holding objects or arrays, each one is unique so that the nested + // iterations never shadow each other. + items int +} + +// contains returns the conditions matching when the JSON value at path in +// doc, or doc itself when path is empty, contains want. +func (sc *sqliteContains) contains(doc, path string, want any) ([]string, error) { + switch want := want.(type) { + case map[string]any: + return sc.containsObject(doc, path, want) + case []any: + return sc.containsArray(doc, path, want) + } + + if path == "" { + path = "$" + } + + var ( + typ = sc.args(doc, path) + value = sc.args(doc, path) + ) + + return []string{ + fmt.Sprintf( + "json_type(%s) IN ('integer', 'real', 'text') AND json_extract(%s) = %s", + typ, + value, + sc.w.RedeemVariable(sqliteScalar(want)), + ), + }, nil +} + +func (sc *sqliteContains) containsObject(doc, path string, obj map[string]any) ([]string, error) { + var cs []string + + // The type of a non empty object below the root is checked by the + // conditions on its keys, which never match a value of another type. + if path == "" || len(obj) == 0 { + cs = append(cs, fmt.Sprintf("json_type(%s) = 'object'", sc.args(doc, path))) + } + + keys := make([]string, 0, len(obj)) + + for key := range obj { + keys = append(keys, key) + } + + slices.Sort(keys) + + for _, key := range keys { + if strings.Contains(key, `"`) { + return nil, errors.Wrapf(errInvalidJSONBKey, "%q", key) + } + + parent := path + + if parent == "" { + parent = "$" + } + + kcs, err := sc.contains(doc, fmt.Sprintf(`%s."%s"`, parent, key), obj[key]) + + if err != nil { + return nil, err + } + + cs = append(cs, kcs...) + } + + return cs, nil +} + +// containsArray matches the scalars of es with a single iteration of the +// array, counting the ones it holds, and every object or array of es with +// an iteration of its own. +func (sc *sqliteContains) containsArray(doc, path string, es []any) ([]string, error) { + cs := []string{fmt.Sprintf("json_type(%s) = 'array'", sc.args(doc, path))} + + scalars, others := splitJSONBScalars(es) + + if len(scalars) > 0 { + want, err := encodeJSONB(scalars) + + if err != nil { + return nil, err + } + + elem := sc.args(doc, path) + + cs = append( + cs, + fmt.Sprintf( + "(SELECT count(DISTINCT jsonb_want.key) FROM json_each(%s) AS "+ + "jsonb_elem, json_each(%s) AS jsonb_want WHERE %s) = %d", + elem, + sc.w.RedeemVariable(want), + sqliteJSONBElementMatch, + len(scalars), + ), + ) + } + + for _, e := range others { + c, err := sc.existsItem(doc, path, e) + + if err != nil { + return nil, err + } + + cs = append(cs, c) + } + + return cs, nil +} + +// containsAny returns the condition matching when the JSON array doc holds +// one of the elements of es. +func (sc *sqliteContains) containsAny(doc string, es []any) (string, error) { + var ( + cs []string + + scalars, others = splitJSONBScalars(es) + ) + + if len(scalars) > 0 { + want, err := encodeJSONB(scalars) + + if err != nil { + return "", err + } + + cs = append( + cs, + fmt.Sprintf( + "EXISTS (SELECT 1 FROM json_each(%s) AS jsonb_elem, json_each(%s) "+ + "AS jsonb_want WHERE %s)", + doc, + sc.w.RedeemVariable(want), + sqliteJSONBElementMatch, + ), + ) + } + + for _, e := range others { + c, err := sc.existsItem(doc, "", e) + + if err != nil { + return "", err + } + + cs = append(cs, c) + } + + if len(cs) == 1 { + return cs[0], nil + } + + return "(" + strings.Join(cs, " OR ") + ")", nil +} + +// existsItem returns the condition matching when the JSON array at path in +// doc holds an element containing want, an object or an array. +func (sc *sqliteContains) existsItem(doc, path string, want any) (string, error) { + sc.items++ + + var ( + array = sc.args(doc, path) + alias = fmt.Sprintf("jsonb_item_%d", sc.items) + typ = "array" + ) + + if _, ok := want.(map[string]any); ok { + typ = "object" + } + + // json_each returns the strings of an array without their quotes, which + // the JSON functions reject as malformed: the other elements are only + // read as NULL, as the order the terms of a WHERE clause are evaluated + // in is not guaranteed. + cs, err := sc.contains( + fmt.Sprintf("(CASE WHEN %[1]s.type = '%[2]s' THEN %[1]s.value END)", alias, typ), + "", + want, + ) + + if err != nil { + return "", err + } + + return fmt.Sprintf( + "EXISTS (SELECT 1 FROM json_each(%s) AS %s WHERE %s)", + array, + alias, + strings.Join(cs, " AND "), + ), nil +} + +// args returns the arguments of a JSON function reading the value at path +// in doc, or doc itself when path is empty. +func (sc *sqliteContains) args(doc, path string) string { + if path == "" { + return doc + } + + return doc + ", " + sc.w.RedeemVariable(path) +} + +func splitJSONBScalars(es []any) ([]any, []any) { + var scalars, others []any + + for _, e := range es { + switch e.(type) { + case map[string]any, []any: + others = append(others, e) + default: + scalars = append(scalars, e) + } + } + + return scalars, others +} + +// sqliteScalar converts a string or a JSON number to the value json_extract +// returns for it. +func sqliteScalar(v any) any { + n, ok := v.(json.Number) + + if !ok { + return v + } + + if i, err := n.Int64(); err == nil { + return i + } + + f, _ := n.Float64() + + return f +} + +// decodeJSONB returns the JSON representation of v, decoded with its +// numbers as json.Number, after checking it holds no booleans and nulls. +func decodeJSONB(v any) (any, error) { + buf, err := json.Marshal(v) + + if err != nil { + return nil, errors.Wrap(err, "cant encode the JSONB value") + } + + var ( + res any + + dec = json.NewDecoder(bytes.NewReader(buf)) + ) + + dec.UseNumber() + + if err := dec.Decode(&res); err != nil { + return nil, errors.Wrap(err, "cant decode the JSONB value") + } + + if res == nil { + return nil, nil + } + + return res, checkJSONB(res) +} + +func decodeJSONBArray(v any) ([]any, error) { + d, err := decodeJSONB(v) + + if err != nil { + return nil, err + } + + switch d := d.(type) { + case nil: + return []any{}, nil + case []any: + return d, nil + } + + return nil, errors.Wrapf(errInvalidJSONBValue, "%T is not an array", v) +} + +func checkJSONB(v any) error { + switch v := v.(type) { + case string, json.Number: + return nil + case map[string]any: + for _, e := range v { + if err := checkJSONB(e); err != nil { + return err + } + } + + return nil + case []any: + for _, e := range v { + if err := checkJSONB(e); err != nil { + return err + } + } + + return nil + } + + return errors.Wrapf(errInvalidJSONBValue, "unexpected %T", v) +} + +func encodeJSONB(v any) (string, error) { + buf, err := json.Marshal(v) + + if err != nil { + return "", errors.Wrap(err, "cant encode the JSONB value") + } + + return string(buf), nil +} diff --git a/x/sqlbuilder/jsonb_predicate_clause_test.go b/x/sqlbuilder/jsonb_predicate_clause_test.go new file mode 100644 index 0000000..9c0ce8a --- /dev/null +++ b/x/sqlbuilder/jsonb_predicate_clause_test.go @@ -0,0 +1,371 @@ +package sqlbuilder + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/upfluence/errors/errtest" + + "github.com/upfluence/sql/backend/static" +) + +const ( + postgresDriver = "postgres" + + skuKey = "sku" + skuPath = `$."sku"` + idPath = `$."id"` + typePath = `$."type"` +) + +type testBoundEntity struct { + Type string `json:"type"` + ID int64 `json:"id"` +} + +// sqliteBoundEntityMatch returns the condition matching an element of the +// entities column containing a testBoundEntity, its arguments being +// numbered from first. +func sqliteBoundEntityMatch(alias string, first int) string { + return fmt.Sprintf( + "EXISTS (SELECT 1 FROM json_each(entities) AS %[1]s WHERE "+ + "json_type((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END)) = 'object' AND "+ + "json_type((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END), $%[2]d) IN ('integer', 'real', 'text') AND "+ + "json_extract((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END), $%[3]d) = $%[4]d AND "+ + "json_type((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END), $%[5]d) IN ('integer', 'real', 'text') AND "+ + "json_extract((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END), $%[6]d) = $%[7]d)", + alias, + first, + first+1, + first+2, + first+3, + first+4, + first+5, + ) +} + +func sqliteBoundEntityArgs(e testBoundEntity) []any { + return []any{idPath, idPath, e.ID, typePath, typePath, e.Type} +} + +func TestJSONBPredicateClause(t *testing.T) { + var ( + tags = Column("tags") + data = Column("data") + entities = Column("entities") + + campaign = testBoundEntity{Type: "campaign", ID: 12} + contribution = testBoundEntity{Type: "contribution", ID: 14} + ) + + for _, tt := range []struct { + name string + haveDriver string + haveClause PredicateClause + want string + wantArgs []any + wantErr errtest.ErrorAssertion + }{ + { + name: "postgres contains any", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny(tags, []any{"a", 1}), + want: "(tags @> $1::jsonb OR tags @> $2::jsonb)", + wantArgs: []any{`["a"]`, `[1]`}, + wantErr: errtest.NoError(), + }, + { + name: "unknown driver contains any", + haveClause: StaticJSONBArrayContainsAny(tags, []string{"z"}), + want: "(tags @> $1::jsonb)", + wantArgs: []any{`["z"]`}, + wantErr: errtest.NoError(), + }, + { + name: "postgres contains any of nothing", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny(tags, []string{}), + want: "1=0", + wantErr: errtest.NoError(), + }, + { + name: "postgres contains all", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAll(tags, []any{"a", 1.5}), + want: "tags @> $1::jsonb", + wantArgs: []any{`["a",1.5]`}, + wantErr: errtest.NoError(), + }, + { + name: "postgres contains all of nil", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAll(tags, []string(nil)), + want: "tags @> $1::jsonb", + wantArgs: []any{`[]`}, + wantErr: errtest.NoError(), + }, + { + name: "postgres object contains", + haveDriver: postgresDriver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{ + "a": map[string]any{"b": "x"}, + "n": 1, + "labels": []string{"t"}, + }, + ), + want: "data @> $1::jsonb", + wantArgs: []any{`{"a":{"b":"x"},"labels":["t"],"n":1}`}, + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains any", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAny(tags, []string{"a", "b"}), + want: "(json_type(tags) = 'array' AND EXISTS (SELECT 1 FROM " + + "json_each(tags) AS jsonb_elem, json_each($1) AS jsonb_want " + + "WHERE " + sqliteJSONBElementMatch + "))", + wantArgs: []any{`["a","b"]`}, + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains any of nothing", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAny(tags, []string{}), + want: "1=0", + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains all", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAll(tags, []int{1, 2}), + want: "(json_type(tags) = 'array' AND (SELECT count(DISTINCT " + + "jsonb_want.key) FROM json_each(tags) AS jsonb_elem, " + + "json_each($1) AS jsonb_want WHERE " + sqliteJSONBElementMatch + + ") = 2)", + wantArgs: []any{`[1,2]`}, + wantErr: errtest.NoError(), + }, + { + name: "sqlite object contains", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{ + "a": map[string]any{"b": "x"}, + "e": map[string]any{}, + "n": 1.5, + "i": 2, + "labels": []string{"t"}, + }, + ), + want: "(json_type(data) = 'object' AND " + + "json_type(data, $1) IN ('integer', 'real', 'text') AND " + + "json_extract(data, $2) = $3 AND " + + "json_type(data, $4) = 'object' AND " + + "json_type(data, $5) IN ('integer', 'real', 'text') AND " + + "json_extract(data, $6) = $7 AND " + + "json_type(data, $8) = 'array' AND (SELECT count(DISTINCT " + + "jsonb_want.key) FROM json_each(data, $9) AS jsonb_elem, " + + "json_each($10) AS jsonb_want WHERE " + sqliteJSONBElementMatch + + ") = 1 AND " + + "json_type(data, $11) IN ('integer', 'real', 'text') AND " + + "json_extract(data, $12) = $13)", + wantArgs: []any{ + `$."a"."b"`, `$."a"."b"`, "x", + `$."e"`, + `$."i"`, `$."i"`, int64(2), + `$."labels"`, `$."labels"`, `["t"]`, + `$."n"`, `$."n"`, 1.5, + }, + wantErr: errtest.NoError(), + }, + { + name: "sqlite object contains nothing", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains(data, struct{}{}), + want: "(json_type(data) = 'object')", + wantErr: errtest.NoError(), + }, + { + name: "postgres contains any object", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny( + entities, + []testBoundEntity{campaign, contribution}, + ), + want: "(entities @> $1::jsonb OR entities @> $2::jsonb)", + wantArgs: []any{ + `[{"id":12,"type":"campaign"}]`, + `[{"id":14,"type":"contribution"}]`, + }, + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains any object", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAny( + entities, + []testBoundEntity{campaign, contribution}, + ), + want: "(json_type(entities) = 'array' AND (" + + sqliteBoundEntityMatch("jsonb_item_1", 1) + " OR " + + sqliteBoundEntityMatch("jsonb_item_2", 7) + "))", + wantArgs: append( + sqliteBoundEntityArgs(campaign), + sqliteBoundEntityArgs(contribution)..., + ), + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains all of scalars, objects and arrays", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAll( + tags, + []any{"a", map[string]any{skuKey: "x"}, []any{1}}, + ), + want: "(json_type(tags) = 'array' AND (SELECT count(DISTINCT " + + "jsonb_want.key) FROM json_each(tags) AS jsonb_elem, " + + "json_each($1) AS jsonb_want WHERE " + sqliteJSONBElementMatch + + ") = 1 AND EXISTS (SELECT 1 FROM json_each(tags) AS jsonb_item_1 " + + "WHERE json_type((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END)) = 'object' AND " + + "json_type((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END), $2) IN ('integer', 'real', 'text') " + + "AND json_extract((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END), $3) = $4) AND EXISTS " + + "(SELECT 1 FROM json_each(tags) AS jsonb_item_2 WHERE " + + "json_type((CASE WHEN jsonb_item_2.type = 'array' THEN jsonb_item_2.value END)) = 'array' AND (SELECT " + + "count(DISTINCT jsonb_want.key) FROM json_each((CASE WHEN jsonb_item_2.type = 'array' THEN jsonb_item_2.value END)) " + + "AS jsonb_elem, json_each($5) AS jsonb_want WHERE " + + sqliteJSONBElementMatch + ") = 1))", + wantArgs: []any{`["a"]`, skuPath, skuPath, "x", `[1]`}, + wantErr: errtest.NoError(), + }, + { + name: "postgres object holding an array of objects", + haveDriver: postgresDriver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{"items": []any{map[string]any{skuKey: "x"}}}, + ), + want: "data @> $1::jsonb", + wantArgs: []any{`{"items":[{"sku":"x"}]}`}, + wantErr: errtest.NoError(), + }, + { + name: "sqlite object holding an array of objects", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{"items": []any{map[string]any{skuKey: "x"}}}, + ), + want: "(json_type(data) = 'object' AND json_type(data, $1) = 'array' " + + "AND EXISTS (SELECT 1 FROM json_each(data, $2) AS jsonb_item_1 " + + "WHERE json_type((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END)) = 'object' AND " + + "json_type((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END), $3) IN ('integer', 'real', 'text') " + + "AND json_extract((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END), $4) = $5))", + wantArgs: []any{`$."items"`, `$."items"`, skuPath, skuPath, "x"}, + wantErr: errtest.NoError(), + }, + { + name: "boolean element", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny(tags, []any{"a", true}), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "null element", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAll(tags, []any{nil}), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "object holding a boolean in an array", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny( + entities, + []any{map[string]any{"active": true}}, + ), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "not an array", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAny(tags, "a"), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "object holding a boolean", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{"a": map[string]any{"b": false}}, + ), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "not an object", + haveDriver: postgresDriver, + haveClause: StaticJSONBObjectContains(data, []string{"a"}), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "sqlite object key holding a double quote", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains(data, map[string]any{`a"b`: 1}), + wantErr: errtest.ErrorCause(errInvalidJSONBKey), + }, + } { + t.Run(tt.name, func(t *testing.T) { + w := queryWriter{driver: tt.haveDriver} + + err := tt.haveClause.WriteTo(&w, nil) + + tt.wantErr.Assert(t, err) + assert.Equal(t, tt.want, w.String()) + assert.Equal(t, tt.wantArgs, w.vs) + }) + } +} + +type driverQueryer struct { + *static.Queryer + + driver string +} + +func (dq driverQueryer) Driver() string { return dq.driver } + +func TestJSONBPredicateClauseDriver(t *testing.T) { + var ( + q static.Queryer + + tags = Column("tags") + qb = QueryBuilder{ + Queryer: driverQueryer{Queryer: &q, driver: sqlite3Driver}, + } + ) + + _, err := qb.PrepareDelete( + DeleteStatement{ + Table: "rows", + WhereClause: JSONBArrayContainsAny(tags), + }, + ).Exec(context.Background(), map[string]any{tags.Binding(): []string{"c"}}) + + require.NoError(t, err) + require.Len(t, q.ExecQueries, 1) + assert.Equal( + t, + static.Query{ + Query: "DELETE FROM rows WHERE (json_type(tags) = 'array' AND " + + "EXISTS (SELECT 1 FROM json_each(tags) AS jsonb_elem, " + + "json_each($1) AS jsonb_want WHERE " + sqliteJSONBElementMatch + + "))", + Args: []any{`["c"]`}, + }, + q.ExecQueries[0], + ) +} diff --git a/x/sqlbuilder/query_builder.go b/x/sqlbuilder/query_builder.go index 82c7ce9..38aa630 100644 --- a/x/sqlbuilder/query_builder.go +++ b/x/sqlbuilder/query_builder.go @@ -15,6 +15,8 @@ type QueryBuilder struct { sql.Queryer } +func (qb *QueryBuilder) driver() string { return sql.DriverOf(qb.Queryer) } + func (qb *QueryBuilder) PrepareSelect(ss SelectStatement) *SelectQueryer { return &SelectQueryer{QueryBuilder: qb, Statement: ss} } @@ -44,7 +46,7 @@ func (qb *QueryBuilder) PrepareDelete(ds DeleteStatement) *DeleteExecer { } type statement interface { - buildQuery(map[string]any) (string, []any, error) + buildQuery(string, map[string]any) (string, []any, error) } type InsertExecer struct { @@ -66,7 +68,7 @@ func (ie *InsertExecer) QueryRow(ctx context.Context, qvs map[string]any) sql.Sc stmt := ie.Statement stmt.isQuery = true - sstmt, vs, err := stmt.buildQuery(qvs) + sstmt, vs, err := stmt.buildQuery(ie.qb.driver(), qvs) if err != nil { return errScanner{error: err} @@ -76,7 +78,7 @@ func (ie *InsertExecer) QueryRow(ctx context.Context, qvs map[string]any) sql.Sc } func (ie *InsertExecer) MultiExec(ctx context.Context, vvs []map[string]any, qvs map[string]any) (sql.Result, error) { - stmt, vs, err := ie.Statement.buildQueries(vvs, qvs) + stmt, vs, err := ie.Statement.buildQueries(ie.qb.driver(), vvs, qvs) if err != nil { return nil, err @@ -94,7 +96,7 @@ func (ie *InsertExecer) MultiQuery(ctx context.Context, vvs []map[string]any, qv stmt := ie.Statement stmt.isQuery = true - sstmt, vs, err := stmt.buildQueries(vvs, qvs) + sstmt, vs, err := stmt.buildQueries(ie.qb.driver(), vvs, qvs) if err != nil { return nil, err @@ -136,7 +138,7 @@ type SelectQueryer struct { } func (sq *SelectQueryer) Query(ctx context.Context, qvs map[string]any) (Cursor, error) { - stmt, vs, ks, err := sq.Statement.buildQuery(qvs) + stmt, vs, ks, err := sq.Statement.buildQuery(sq.QueryBuilder.driver(), qvs) if err != nil { return nil, err @@ -152,7 +154,7 @@ func (sq *SelectQueryer) Query(ctx context.Context, qvs map[string]any) (Cursor, } func (sq *SelectQueryer) QueryRow(ctx context.Context, qvs map[string]any) Scanner { - stmt, vs, ks, err := sq.Statement.buildQuery(qvs) + stmt, vs, ks, err := sq.Statement.buildQuery(sq.QueryBuilder.driver(), qvs) if err != nil { return ErrScanner{Err: err} @@ -165,6 +167,10 @@ type QueryWriter interface { io.Writer RedeemVariable(any) string + + // Driver returns the driver the query is built for, or an empty string + // when the queryer of the QueryBuilder does not expose it. + Driver() string } type queryWriter struct { @@ -172,8 +178,12 @@ type queryWriter struct { i int vs []any + + driver string } +func (qw *queryWriter) Driver() string { return qw.driver } + func (qw *queryWriter) RedeemVariable(v any) string { qw.i++ qw.vs = append(qw.vs, v) diff --git a/x/sqlbuilder/select_statement.go b/x/sqlbuilder/select_statement.go index 4cd203e..c8b371a 100644 --- a/x/sqlbuilder/select_statement.go +++ b/x/sqlbuilder/select_statement.go @@ -51,10 +51,11 @@ func writeSelectClause(c Marker, qw *queryWriter, vs map[string]any) error { return nil } -func (ss SelectStatement) buildQuery(vs map[string]any) (string, []any, []string, error) { +func (ss SelectStatement) buildQuery(driver string, vs map[string]any) (string, []any, []string, error) { var ( - qw queryWriter bindings []string + + qw = queryWriter{driver: driver} ) if len(ss.SelectClauses) == 0 { diff --git a/x/sqlbuilder/select_statement_test.go b/x/sqlbuilder/select_statement_test.go index 661bb28..3e140db 100644 --- a/x/sqlbuilder/select_statement_test.go +++ b/x/sqlbuilder/select_statement_test.go @@ -411,7 +411,7 @@ func TestSelectQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, _, err := tt.ss.Clone().buildQuery(tt.vs) + stmt, args, _, err := tt.ss.Clone().buildQuery("", tt.vs) assert.Equal(t, tt.err, err) assert.Equal(t, tt.stmt, stmt) diff --git a/x/sqlbuilder/update_statement.go b/x/sqlbuilder/update_statement.go index f4bff54..bf8cec2 100644 --- a/x/sqlbuilder/update_statement.go +++ b/x/sqlbuilder/update_statement.go @@ -52,8 +52,8 @@ func writeUpdateClauses(fs []Marker, qw QueryWriter, vs map[string]any) error { return nil } -func (us UpdateStatement) buildQuery(vs map[string]any) (string, []any, error) { - var qw queryWriter +func (us UpdateStatement) buildQuery(driver string, vs map[string]any) (string, []any, error) { + qw := queryWriter{driver: driver} if len(us.Fields) == 0 { return "", nil, errNoMarkers diff --git a/x/sqlbuilder/update_statement_test.go b/x/sqlbuilder/update_statement_test.go index b5d5f17..836bc32 100644 --- a/x/sqlbuilder/update_statement_test.go +++ b/x/sqlbuilder/update_statement_test.go @@ -58,7 +58,7 @@ func TestUpdateQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, err := tt.us.Clone().buildQuery(tt.vs) + stmt, args, err := tt.us.Clone().buildQuery("", tt.vs) assert.Equal(t, tt.err, err) assert.Equal(t, tt.stmt, stmt)