From bdc45c864bbecb4b0864dfe0789b82e3c35b395e Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Sun, 1 Aug 2021 18:57:38 +0500 Subject: [PATCH 01/14] aliases and functions --- query/query.go | 1 + sql.go | 28 +++++++++++-- sql_test.go | 111 ++++++++++++++++++++++++++++++------------------- 3 files changed, 94 insertions(+), 46 deletions(-) diff --git a/query/query.go b/query/query.go index 4cb28c4..ed9cd46 100644 --- a/query/query.go +++ b/query/query.go @@ -8,6 +8,7 @@ type Query struct { Updates map[string]string Inserts [][]string Fields []string // Used for SELECT (i.e. SELECTed field names) and INSERT (INSERTEDed field names) + Aliases []string // Used for SELECT (i.e. SELECTed field_name AS alias_name) } // Type is the type of SQL query, e.g. SELECT/UPDATE diff --git a/sql.go b/sql.go index 7da4ea9..9af43d2 100644 --- a/sql.go +++ b/sql.go @@ -120,8 +120,21 @@ func (p *parser) doParse() (query.Query, error) { } p.query.Fields = append(p.query.Fields, identifier) p.pop() - maybeFrom := p.peek() - if strings.ToUpper(maybeFrom) == "FROM" { + maybeFrom := strings.ToUpper(p.peek()) + if maybeFrom == "AS" { + // alias + p.pop() + alias := p.peek() + if !isIdentifierOrAsterisk(alias) { + return p.query, fmt.Errorf("at SELECT: expected alias (AS) for %s", identifier) + } + p.query.Aliases = append(p.query.Aliases, alias) + p.pop() + maybeFrom = p.peek() + } else { + p.query.Aliases = append(p.query.Aliases, "") + } + if maybeFrom == "FROM" { p.step = stepSelectFrom continue } @@ -369,7 +382,7 @@ func (p *parser) popWhitespace() { } var reservedWords = []string{ - "(", ")", ">=", "<=", "!=", ",", "=", ">", "<", "SELECT", "INSERT INTO", "VALUES", "UPDATE", "DELETE FROM", + "(", ")", ">=", "<=", "!=", ",", "=", ">", "<", "AS", "SELECT", "INSERT INTO", "VALUES", "UPDATE", "DELETE FROM", "WHERE", "FROM", "SET", } @@ -404,6 +417,12 @@ func (p *parser) peekQuotedStringWithLength() (string, int) { func (p *parser) peekIdentifierWithLength() (string, int) { for i := p.i; i < len(p.sql); i++ { if matched, _ := regexp.MatchString(`[a-zA-Z0-9_*]`, string(p.sql[i])); !matched { + if p.sql[i] == '(' { + // detect function + if end := strings.IndexByte(p.sql[i+1:], ')'); end >= 0 { + i += end + 2 + } + } return p.sql[p.i:i], len(p.sql[p.i:i]) } } @@ -444,6 +463,9 @@ func (p *parser) validate() error { } } } + if p.query.Type == query.Select && len(p.query.Fields) != len(p.query.Aliases) { + return fmt.Errorf("fileds and aliases count mismatch") + } return nil } diff --git a/sql_test.go b/sql_test.go index f15cde3..2a76371 100644 --- a/sql_test.go +++ b/sql_test.go @@ -28,58 +28,82 @@ type output struct { func TestSQL(t *testing.T) { ts := []testCase{ - { - Name: "empty query fails", - SQL: "", - Expected: query.Query{}, - Err: fmt.Errorf("query type cannot be empty"), - }, - { - Name: "SELECT without FROM fails", - SQL: "SELECT", - Expected: query.Query{Type: query.Select}, - Err: fmt.Errorf("table name cannot be empty"), - }, - { - Name: "SELECT without fields fails", - SQL: "SELECT FROM 'a'", - Expected: query.Query{Type: query.Select}, - Err: fmt.Errorf("at SELECT: expected field to SELECT"), - }, - { - Name: "SELECT with comma and empty field fails", - SQL: "SELECT b, FROM 'a'", - Expected: query.Query{Type: query.Select}, - Err: fmt.Errorf("at SELECT: expected field to SELECT"), - }, - { - Name: "SELECT works", - SQL: "SELECT a FROM 'b'", - Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}}, + // { + // Name: "empty query fails", + // SQL: "", + // Expected: query.Query{}, + // Err: fmt.Errorf("query type cannot be empty"), + // }, + // { + // Name: "SELECT without FROM fails", + // SQL: "SELECT", + // Expected: query.Query{Type: query.Select}, + // Err: fmt.Errorf("table name cannot be empty"), + // }, + // { + // Name: "SELECT without fields fails", + // SQL: "SELECT FROM 'a'", + // Expected: query.Query{Type: query.Select}, + // Err: fmt.Errorf("at SELECT: expected field to SELECT"), + // }, + // { + // Name: "SELECT with comma and empty field fails", + // SQL: "SELECT b, FROM 'a'", + // Expected: query.Query{Type: query.Select}, + // Err: fmt.Errorf("at SELECT: expected field to SELECT"), + // { + // Name: "SELECT with incomplete alias fails", + // SQL: "SELECT a AS", + // Expected: query.Query{Type: query.Select}, + // Err: fmt.Errorf("at SELECT: expected alias (AS) for a"), + // }, + // }, + // { + // Name: "SELECT version() as version", + // SQL: "SELECT version() as version", + // Expected: query.Query{Type: query.Select, Fields: []string{"version() as version"}}, + // Err: nil, + // }, + // { + // Name: "SELECT works", + // SQL: "SELECT a FROM 'b'", + // Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}}, + // Err: nil, + // }, + // { + // Name: "SELECT with alias works", + // SQL: "SELECT a AS text FROM 'b'", + // Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}, Aliases: []string{"text"}}, + // Err: nil, + // }, + { + Name: "SELECT with alias works", + SQL: "SELECT version(a) AS version FROM 'b'", + Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"version(a)"}, Aliases: []string{"version"}}, Err: nil, }, { Name: "SELECT works with lowercase", SQL: "select a fRoM 'b'", - Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}}, + Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}, Aliases: []string{""}}, Err: nil, }, { Name: "SELECT many fields works", SQL: "SELECT a, c, d FROM 'b'", - Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a", "c", "d"}}, + Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}}, Err: nil, }, { Name: "SELECT with empty WHERE fails", SQL: "SELECT a, c, d FROM 'b' WHERE", - Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a", "c", "d"}}, + Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}}, Err: fmt.Errorf("at WHERE: empty WHERE clause"), }, { Name: "SELECT with WHERE with only operand fails", SQL: "SELECT a, c, d FROM 'b' WHERE a", - Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a", "c", "d"}}, + Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}}, Err: fmt.Errorf("at WHERE: condition without operator"), }, { @@ -88,7 +112,7 @@ func TestSQL(t *testing.T) { Expected: query.Query{ Type: query.Select, TableName: "b", - Fields: []string{"a", "c", "d"}, + Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ {Operand1: "a", Operand1IsField: true, Operator: query.Eq, Operand2: "", Operand2IsField: false}, }, @@ -101,7 +125,7 @@ func TestSQL(t *testing.T) { Expected: query.Query{ Type: query.Select, TableName: "b", - Fields: []string{"a", "c", "d"}, + Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ {Operand1: "a", Operand1IsField: true, Operator: query.Lt, Operand2: "1", Operand2IsField: false}, }, @@ -114,7 +138,7 @@ func TestSQL(t *testing.T) { Expected: query.Query{ Type: query.Select, TableName: "b", - Fields: []string{"a", "c", "d"}, + Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ {Operand1: "a", Operand1IsField: true, Operator: query.Lte, Operand2: "1", Operand2IsField: false}, }, @@ -127,7 +151,7 @@ func TestSQL(t *testing.T) { Expected: query.Query{ Type: query.Select, TableName: "b", - Fields: []string{"a", "c", "d"}, + Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ {Operand1: "a", Operand1IsField: true, Operator: query.Gt, Operand2: "1", Operand2IsField: false}, }, @@ -140,7 +164,7 @@ func TestSQL(t *testing.T) { Expected: query.Query{ Type: query.Select, TableName: "b", - Fields: []string{"a", "c", "d"}, + Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ {Operand1: "a", Operand1IsField: true, Operator: query.Gte, Operand2: "1", Operand2IsField: false}, }, @@ -153,7 +177,7 @@ func TestSQL(t *testing.T) { Expected: query.Query{ Type: query.Select, TableName: "b", - Fields: []string{"a", "c", "d"}, + Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ {Operand1: "a", Operand1IsField: true, Operator: query.Ne, Operand2: "1", Operand2IsField: false}, }, @@ -166,7 +190,7 @@ func TestSQL(t *testing.T) { Expected: query.Query{ Type: query.Select, TableName: "b", - Fields: []string{"a", "c", "d"}, + Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ {Operand1: "a", Operand1IsField: true, Operator: query.Ne, Operand2: "b", Operand2IsField: true}, }, @@ -180,6 +204,7 @@ func TestSQL(t *testing.T) { Type: query.Select, TableName: "b", Fields: []string{"*"}, + Aliases: []string{""}, Conditions: nil, }, Err: nil, @@ -188,9 +213,9 @@ func TestSQL(t *testing.T) { Name: "SELECT a, * works", SQL: "SELECT a, * FROM 'b'", Expected: query.Query{ - Type: query.Select, - TableName: "b", - Fields: []string{"a", "*"}, + Type: query.Select, + TableName: "b", + Fields: []string{"a", "*"}, Aliases: []string{"", ""}, Conditions: nil, }, Err: nil, @@ -201,7 +226,7 @@ func TestSQL(t *testing.T) { Expected: query.Query{ Type: query.Select, TableName: "b", - Fields: []string{"a", "c", "d"}, + Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ {Operand1: "a", Operand1IsField: true, Operator: query.Ne, Operand2: "1", Operand2IsField: false}, {Operand1: "b", Operand1IsField: true, Operator: query.Eq, Operand2: "2", Operand2IsField: false}, From 5104f0fe1f3257b4bece4dc614182e1533974baf Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Sun, 1 Aug 2021 19:14:08 +0500 Subject: [PATCH 02/14] select without from (clickhouse sql dialect) --- README.md | 53 +++++++++++++++++++++++++++++ sql.go | 2 +- sql_test.go | 96 ++++++++++++++++++++++++++--------------------------- 3 files changed, 102 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index e340217..3c5d5d8 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,21 @@ func main() { ``` +### Example: SELECT version() as version + +``` +query, err := sqlparser.Parse(`SELECT version() as version`) + +query.Query { + Type: Select + TableName: + Conditions: [] + Updates: map[] + Inserts: [] + Fields: [version()] +} +``` + ### Example: SELECT works ``` @@ -37,6 +52,36 @@ query.Query { } ``` +### Example: SELECT with alias works + +``` +query, err := sqlparser.Parse(`SELECT a AS text FROM 'b'`) + +query.Query { + Type: Select + TableName: b + Conditions: [] + Updates: map[] + Inserts: [] + Fields: [a] +} +``` + +### Example: SELECT with alias works + +``` +query, err := sqlparser.Parse(`SELECT version(a) AS version FROM 'b'`) + +query.Query { + Type: Select + TableName: b + Conditions: [] + Updates: map[] + Inserts: [] + Fields: [version(a)] +} +``` + ### Example: SELECT works with lowercase ``` @@ -476,6 +521,14 @@ query, err := sqlparser.Parse(`SELECT b, FROM 'a'`) at SELECT: expected field to SELECT ``` +### Example: SELECT with incomplete alias fails + +``` +query, err := sqlparser.Parse(`SELECT a AS`) + +at SELECT: expected alias (AS) for a +``` + ### Example: SELECT with empty WHERE fails ``` diff --git a/sql.go b/sql.go index 9af43d2..9f915d9 100644 --- a/sql.go +++ b/sql.go @@ -436,7 +436,7 @@ func (p *parser) validate() error { if p.query.Type == query.UnknownType { return fmt.Errorf("query type cannot be empty") } - if p.query.TableName == "" { + if (p.query.Type != query.Select || len(p.query.Fields) == 0) && p.query.TableName == "" { return fmt.Errorf("table name cannot be empty") } if len(p.query.Conditions) == 0 && (p.query.Type == query.Update || p.query.Type == query.Delete) { diff --git a/sql_test.go b/sql_test.go index 2a76371..a650026 100644 --- a/sql_test.go +++ b/sql_test.go @@ -28,54 +28,54 @@ type output struct { func TestSQL(t *testing.T) { ts := []testCase{ - // { - // Name: "empty query fails", - // SQL: "", - // Expected: query.Query{}, - // Err: fmt.Errorf("query type cannot be empty"), - // }, - // { - // Name: "SELECT without FROM fails", - // SQL: "SELECT", - // Expected: query.Query{Type: query.Select}, - // Err: fmt.Errorf("table name cannot be empty"), - // }, - // { - // Name: "SELECT without fields fails", - // SQL: "SELECT FROM 'a'", - // Expected: query.Query{Type: query.Select}, - // Err: fmt.Errorf("at SELECT: expected field to SELECT"), - // }, - // { - // Name: "SELECT with comma and empty field fails", - // SQL: "SELECT b, FROM 'a'", - // Expected: query.Query{Type: query.Select}, - // Err: fmt.Errorf("at SELECT: expected field to SELECT"), - // { - // Name: "SELECT with incomplete alias fails", - // SQL: "SELECT a AS", - // Expected: query.Query{Type: query.Select}, - // Err: fmt.Errorf("at SELECT: expected alias (AS) for a"), - // }, - // }, - // { - // Name: "SELECT version() as version", - // SQL: "SELECT version() as version", - // Expected: query.Query{Type: query.Select, Fields: []string{"version() as version"}}, - // Err: nil, - // }, - // { - // Name: "SELECT works", - // SQL: "SELECT a FROM 'b'", - // Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}}, - // Err: nil, - // }, - // { - // Name: "SELECT with alias works", - // SQL: "SELECT a AS text FROM 'b'", - // Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}, Aliases: []string{"text"}}, - // Err: nil, - // }, + { + Name: "empty query fails", + SQL: "", + Expected: query.Query{}, + Err: fmt.Errorf("query type cannot be empty"), + }, + { + Name: "SELECT without FROM fails", + SQL: "SELECT", + Expected: query.Query{Type: query.Select}, + Err: fmt.Errorf("table name cannot be empty"), + }, + { + Name: "SELECT without fields fails", + SQL: "SELECT FROM 'a'", + Expected: query.Query{Type: query.Select}, + Err: fmt.Errorf("at SELECT: expected field to SELECT"), + }, + { + Name: "SELECT with comma and empty field fails", + SQL: "SELECT b, FROM 'a'", + Expected: query.Query{Type: query.Select}, + Err: fmt.Errorf("at SELECT: expected field to SELECT"), + }, + { + Name: "SELECT with incomplete alias fails", + SQL: "SELECT a AS", + Expected: query.Query{Type: query.Select}, + Err: fmt.Errorf("at SELECT: expected alias (AS) for a"), + }, + { + Name: "SELECT version() as version", + SQL: "SELECT version() as version", + Expected: query.Query{Type: query.Select, Fields: []string{"version()"}, Aliases: []string{"version"}}, + Err: nil, + }, + { + Name: "SELECT works", + SQL: "SELECT a FROM 'b'", + Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}, Aliases: []string{""}}, + Err: nil, + }, + { + Name: "SELECT with alias works", + SQL: "SELECT a AS text FROM 'b'", + Expected: query.Query{Type: query.Select, TableName: "b", Fields: []string{"a"}, Aliases: []string{"text"}}, + Err: nil, + }, { Name: "SELECT with alias works", SQL: "SELECT version(a) AS version FROM 'b'", From 6889c2c833f41a5567c97a32de41db639e894fd6 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Sun, 1 Aug 2021 19:35:24 +0500 Subject: [PATCH 03/14] select/insert parse benchmark --- sql_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sql_test.go b/sql_test.go index a650026..9545ee7 100644 --- a/sql_test.go +++ b/sql_test.go @@ -474,6 +474,26 @@ func TestSQL(t *testing.T) { createReadme(output) } +func BenchmarkSQLSelect(b *testing.B) { + sql := "SELECT a AS text FROM 'b'" + for i := 0; i < b.N; i++ { + q, err := Parse(sql) + if err != nil { + b.Errorf("Error should have been %v: %v", err, q) + } + } +} + +func BenchmarkSQLInsert(b *testing.B) { + sql := "INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' )" + for i := 0; i < b.N; i++ { + q, err := Parse(sql) + if err != nil { + b.Errorf("Error should have been %v: %v", err, q) + } + } +} + func createReadme(out output) { content, err := ioutil.ReadFile("README.template") if err != nil { From 7863d2424b3674bda58edf4d7f87f9f04b9986de Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Sun, 1 Aug 2021 20:07:00 +0500 Subject: [PATCH 04/14] optimize perfomance #1: optimize identifier scan --- sql.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sql.go b/sql.go index 9f915d9..1465bf2 100644 --- a/sql.go +++ b/sql.go @@ -416,7 +416,12 @@ func (p *parser) peekQuotedStringWithLength() (string, int) { func (p *parser) peekIdentifierWithLength() (string, int) { for i := p.i; i < len(p.sql); i++ { - if matched, _ := regexp.MatchString(`[a-zA-Z0-9_*]`, string(p.sql[i])); !matched { + isIdentifierSymbol := (p.sql[i] >= 'a' && p.sql[i] <= 'z') || + (p.sql[i] >= 'A' && p.sql[i] <= 'Z') || + (p.sql[i] >= '0' && p.sql[i] <= '9') || + p.sql[i] == '*' || + p.sql[i] == '_' + if !isIdentifierSymbol { if p.sql[i] == '(' { // detect function if end := strings.IndexByte(p.sql[i+1:], ')'); end >= 0 { @@ -478,18 +483,19 @@ func (p *parser) logError() { fmt.Println(p.err) } +var regexIdentifier = regexp.MustCompile("[a-zA-Z_][a-zA-Z_0-9]*") + func isIdentifier(s string) bool { for _, rw := range reservedWords { if strings.ToUpper(s) == rw { return false } } - matched, _ := regexp.MatchString("[a-zA-Z_][a-zA-Z_0-9]*", s) - return matched + return regexIdentifier.MatchString(s) } func isIdentifierOrAsterisk(s string) bool { - return isIdentifier(s) || s == "*" + return s == "*" || isIdentifier(s) } func min(a, b int) int { From 017cc14ad5554a3cebe9ad883571e380440f2919 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Sun, 1 Aug 2021 20:40:19 +0500 Subject: [PATCH 05/14] extend sql parse benchmark --- sql_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql_test.go b/sql_test.go index 9545ee7..70e2655 100644 --- a/sql_test.go +++ b/sql_test.go @@ -475,7 +475,7 @@ func TestSQL(t *testing.T) { } func BenchmarkSQLSelect(b *testing.B) { - sql := "SELECT a AS text FROM 'b'" + sql := "SELECT a AS text FROM 'b' WHERE c = 'c' AND d = 'd'" for i := 0; i < b.N; i++ { q, err := Parse(sql) if err != nil { From 70f490eb3f18e3e8f5461dbd223f0a8f467506d2 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Sun, 1 Aug 2021 22:10:17 +0500 Subject: [PATCH 06/14] optimize perfomance #2: remove scan on pop (use peek results) --- sql.go | 147 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 90 insertions(+), 57 deletions(-) diff --git a/sql.go b/sql.go index 1465bf2..c25a44e 100644 --- a/sql.go +++ b/sql.go @@ -32,7 +32,8 @@ func ParseMany(sqls []string) ([]query.Query, error) { } func parse(sql string) (query.Query, error) { - return (&parser{0, strings.TrimSpace(sql), stepType, query.Query{}, nil, ""}).parse() + sql = strings.TrimSpace(sql) + return (&parser{0, 0, "", sql, strings.ToUpper(sql), stepType, query.Query{}, nil, ""}).parse() } type step int @@ -68,7 +69,10 @@ const ( type parser struct { i int + len int + peeked string sql string + sqlUpper string step step query query.Query err error @@ -92,45 +96,43 @@ func (p *parser) doParse() (query.Query, error) { } switch p.step { case stepType: - switch strings.ToUpper(p.peek()) { + s := p.peek(true) + switch s { case "SELECT": p.query.Type = query.Select - p.pop() p.step = stepSelectField case "INSERT INTO": p.query.Type = query.Insert - p.pop() p.step = stepInsertTable case "UPDATE": p.query.Type = query.Update p.query.Updates = map[string]string{} - p.pop() p.step = stepUpdateTable case "DELETE FROM": p.query.Type = query.Delete - p.pop() p.step = stepDeleteFromTable default: return p.query, fmt.Errorf("invalid query type") } + p.pop() case stepSelectField: - identifier := p.peek() + identifier := p.peek(false) if !isIdentifierOrAsterisk(identifier) { return p.query, fmt.Errorf("at SELECT: expected field to SELECT") } p.query.Fields = append(p.query.Fields, identifier) p.pop() - maybeFrom := strings.ToUpper(p.peek()) + maybeFrom := p.peek(true) if maybeFrom == "AS" { // alias p.pop() - alias := p.peek() + alias := p.peek(false) if !isIdentifierOrAsterisk(alias) { return p.query, fmt.Errorf("at SELECT: expected alias (AS) for %s", identifier) } p.query.Aliases = append(p.query.Aliases, alias) p.pop() - maybeFrom = p.peek() + maybeFrom = p.peek(true) } else { p.query.Aliases = append(p.query.Aliases, "") } @@ -140,21 +142,21 @@ func (p *parser) doParse() (query.Query, error) { } p.step = stepSelectComma case stepSelectComma: - commaRWord := p.peek() + commaRWord := p.peek(false) if commaRWord != "," { return p.query, fmt.Errorf("at SELECT: expected comma or FROM") } p.pop() p.step = stepSelectField case stepSelectFrom: - fromRWord := p.peek() - if strings.ToUpper(fromRWord) != "FROM" { + fromRWord := p.peek(true) + if fromRWord != "FROM" { return p.query, fmt.Errorf("at SELECT: expected FROM") } p.pop() p.step = stepSelectFromTable case stepSelectFromTable: - tableName := p.peek() + tableName := p.peek(false) if len(tableName) == 0 { return p.query, fmt.Errorf("at SELECT: expected quoted table name") } @@ -162,7 +164,7 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepWhere case stepInsertTable: - tableName := p.peek() + tableName := p.peek(false) if len(tableName) == 0 { return p.query, fmt.Errorf("at INSERT INTO: expected quoted table name") } @@ -170,7 +172,7 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepInsertFieldsOpeningParens case stepDeleteFromTable: - tableName := p.peek() + tableName := p.peek(false) if len(tableName) == 0 { return p.query, fmt.Errorf("at DELETE FROM: expected quoted table name") } @@ -178,7 +180,7 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepWhere case stepUpdateTable: - tableName := p.peek() + tableName := p.peek(false) if len(tableName) == 0 { return p.query, fmt.Errorf("at UPDATE: expected quoted table name") } @@ -186,14 +188,14 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepUpdateSet case stepUpdateSet: - setRWord := p.peek() + setRWord := p.peek(true) if setRWord != "SET" { return p.query, fmt.Errorf("at UPDATE: expected 'SET'") } p.pop() p.step = stepUpdateField case stepUpdateField: - identifier := p.peek() + identifier := p.peek(false) if !isIdentifier(identifier) { return p.query, fmt.Errorf("at UPDATE: expected at least one field to update") } @@ -201,42 +203,42 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepUpdateEquals case stepUpdateEquals: - equalsRWord := p.peek() + equalsRWord := p.peek(false) if equalsRWord != "=" { return p.query, fmt.Errorf("at UPDATE: expected '='") } p.pop() p.step = stepUpdateValue case stepUpdateValue: - quotedValue, ln := p.peekQuotedStringWithLength() - if ln == 0 { + quotedValue := p.peekQuotedString(false) + if p.len == 0 { return p.query, fmt.Errorf("at UPDATE: expected quoted value") } p.query.Updates[p.nextUpdateField] = quotedValue p.nextUpdateField = "" p.pop() - maybeWhere := p.peek() - if strings.ToUpper(maybeWhere) == "WHERE" { + maybeWhere := p.peek(true) + if maybeWhere == "WHERE" { p.step = stepWhere continue } p.step = stepUpdateComma case stepUpdateComma: - commaRWord := p.peek() + commaRWord := p.peek(false) if commaRWord != "," { return p.query, fmt.Errorf("at UPDATE: expected ','") } p.pop() p.step = stepUpdateField case stepWhere: - whereRWord := p.peek() - if strings.ToUpper(whereRWord) != "WHERE" { + whereRWord := p.peek(true) + if whereRWord != "WHERE" { return p.query, fmt.Errorf("expected WHERE") } p.pop() p.step = stepWhereField case stepWhereField: - identifier := p.peek() + identifier := p.peek(false) if !isIdentifier(identifier) { return p.query, fmt.Errorf("at WHERE: expected field") } @@ -244,7 +246,7 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepWhereOperator case stepWhereOperator: - operator := p.peek() + operator := p.peek(false) currentCondition := p.query.Conditions[len(p.query.Conditions)-1] switch operator { case "=": @@ -267,13 +269,13 @@ func (p *parser) doParse() (query.Query, error) { p.step = stepWhereValue case stepWhereValue: currentCondition := p.query.Conditions[len(p.query.Conditions)-1] - identifier := p.peek() + identifier := p.peek(false) if isIdentifier(identifier) { currentCondition.Operand2 = identifier currentCondition.Operand2IsField = true } else { - quotedValue, ln := p.peekQuotedStringWithLength() - if ln == 0 { + quotedValue := p.peekQuotedString(false) + if p.len == 0 { return p.query, fmt.Errorf("at WHERE: expected quoted value") } currentCondition.Operand2 = quotedValue @@ -283,21 +285,21 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepWhereAnd case stepWhereAnd: - andRWord := p.peek() - if strings.ToUpper(andRWord) != "AND" { + andRWord := p.peek(true) + if andRWord != "AND" { return p.query, fmt.Errorf("expected AND") } p.pop() p.step = stepWhereField case stepInsertFieldsOpeningParens: - openingParens := p.peek() + openingParens := p.peek(false) if len(openingParens) != 1 || openingParens != "(" { return p.query, fmt.Errorf("at INSERT INTO: expected opening parens") } p.pop() p.step = stepInsertFields case stepInsertFields: - identifier := p.peek() + identifier := p.peek(false) if !isIdentifier(identifier) { return p.query, fmt.Errorf("at INSERT INTO: expected at least one field to insert") } @@ -305,7 +307,7 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepInsertFieldsCommaOrClosingParens case stepInsertFieldsCommaOrClosingParens: - commaOrClosingParens := p.peek() + commaOrClosingParens := p.peek(false) if commaOrClosingParens != "," && commaOrClosingParens != ")" { return p.query, fmt.Errorf("at INSERT INTO: expected comma or closing parens") } @@ -316,14 +318,14 @@ func (p *parser) doParse() (query.Query, error) { } p.step = stepInsertValuesRWord case stepInsertValuesRWord: - valuesRWord := p.peek() - if strings.ToUpper(valuesRWord) != "VALUES" { + valuesRWord := p.peek(true) + if valuesRWord != "VALUES" { return p.query, fmt.Errorf("at INSERT INTO: expected 'VALUES'") } p.pop() p.step = stepInsertValuesOpeningParens case stepInsertValuesOpeningParens: - openingParens := p.peek() + openingParens := p.peek(false) if openingParens != "(" { return p.query, fmt.Errorf("at INSERT INTO: expected opening parens") } @@ -331,15 +333,15 @@ func (p *parser) doParse() (query.Query, error) { p.pop() p.step = stepInsertValues case stepInsertValues: - quotedValue, ln := p.peekQuotedStringWithLength() - if ln == 0 { + quotedValue := p.peekQuotedString(false) + if p.len == 0 { return p.query, fmt.Errorf("at INSERT INTO: expected quoted value") } p.query.Inserts[len(p.query.Inserts)-1] = append(p.query.Inserts[len(p.query.Inserts)-1], quotedValue) p.pop() p.step = stepInsertValuesCommaOrClosingParens case stepInsertValuesCommaOrClosingParens: - commaOrClosingParens := p.peek() + commaOrClosingParens := p.peek(false) if commaOrClosingParens != "," && commaOrClosingParens != ")" { return p.query, fmt.Errorf("at INSERT INTO: expected comma or closing parens") } @@ -354,8 +356,8 @@ func (p *parser) doParse() (query.Query, error) { } p.step = stepInsertValuesCommaBeforeOpeningParens case stepInsertValuesCommaBeforeOpeningParens: - commaRWord := p.peek() - if strings.ToUpper(commaRWord) != "," { + commaRWord := p.peek(false) + if commaRWord != "," { return p.query, fmt.Errorf("at INSERT INTO: expected comma") } p.pop() @@ -364,18 +366,35 @@ func (p *parser) doParse() (query.Query, error) { } } -func (p *parser) peek() string { - peeked, _ := p.peekWithLength() - return peeked +func (p *parser) peek(upper bool) string { + p.peeked, p.len = p.peekWithLength(upper) + return p.peeked +} + +func (p *parser) peekQuotedString(upper bool) string { + p.peeked, p.len = p.peekQuotedStringWithLength(upper) + return p.peeked +} + +func (p *parser) peekIdentifier(upper bool) string { + p.peeked, p.len = p.peekQuotedStringWithLength(upper) + return p.peeked } func (p *parser) pop() string { - peeked, len := p.peekWithLength() - p.i += len + peeked := p.peeked + p.peeked = "" + p.i += p.len + p.len = 0 p.popWhitespace() return peeked } +func (p *parser) popWithLength(len int) { + p.i += len + p.popWhitespace() +} + func (p *parser) popWhitespace() { for ; p.i < len(p.sql) && p.sql[p.i] == ' '; p.i++ { } @@ -386,35 +405,42 @@ var reservedWords = []string{ "WHERE", "FROM", "SET", } -func (p *parser) peekWithLength() (string, int) { +func (p *parser) peekWithLength(upper bool) (string, int) { if p.i >= len(p.sql) { return "", 0 } for _, rWord := range reservedWords { - token := strings.ToUpper(p.sql[p.i:min(len(p.sql), p.i+len(rWord))]) + token := p.sqlUpper[p.i:min(len(p.sqlUpper), p.i+len(rWord))] if token == rWord { + if !upper { + token = p.sql[p.i:min(len(p.sql), p.i+len(rWord))] + } + return token, len(token) } } if p.sql[p.i] == '\'' { // Quoted string - return p.peekQuotedStringWithLength() + return p.peekQuotedStringWithLength(upper) } - return p.peekIdentifierWithLength() + return p.peekIdentifierWithLength(upper) } -func (p *parser) peekQuotedStringWithLength() (string, int) { +func (p *parser) peekQuotedStringWithLength(upper bool) (string, int) { if len(p.sql) < p.i || p.sql[p.i] != '\'' { return "", 0 } for i := p.i + 1; i < len(p.sql); i++ { if p.sql[i] == '\'' && p.sql[i-1] != '\\' { + if upper { + return p.sqlUpper[p.i+1 : i], len(p.sqlUpper[p.i+1:i]) + 2 // +2 for the two quotes + } return p.sql[p.i+1 : i], len(p.sql[p.i+1:i]) + 2 // +2 for the two quotes } } return "", 0 } -func (p *parser) peekIdentifierWithLength() (string, int) { +func (p *parser) peekIdentifierWithLength(upper bool) (string, int) { for i := p.i; i < len(p.sql); i++ { isIdentifierSymbol := (p.sql[i] >= 'a' && p.sql[i] <= 'z') || (p.sql[i] >= 'A' && p.sql[i] <= 'Z') || @@ -428,9 +454,15 @@ func (p *parser) peekIdentifierWithLength() (string, int) { i += end + 2 } } + if upper { + return p.sqlUpper[p.i:i], len(p.sqlUpper[p.i:i]) + } return p.sql[p.i:i], len(p.sql[p.i:i]) } } + if upper { + return p.sqlUpper[p.i:], len(p.sqlUpper[p.i:]) + } return p.sql[p.i:], len(p.sql[p.i:]) } @@ -486,8 +518,9 @@ func (p *parser) logError() { var regexIdentifier = regexp.MustCompile("[a-zA-Z_][a-zA-Z_0-9]*") func isIdentifier(s string) bool { + u := strings.ToUpper(s) for _, rw := range reservedWords { - if strings.ToUpper(s) == rw { + if u == rw { return false } } From 72011512b678a231b7d47fa27d62d871ee101cb6 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Sun, 1 Aug 2021 23:59:50 +0500 Subject: [PATCH 07/14] optimize perfomance #3: remove slice in Parse --- sql.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/sql.go b/sql.go index c25a44e..a553302 100644 --- a/sql.go +++ b/sql.go @@ -9,12 +9,13 @@ import ( ) // Parse takes a string representing a SQL query and parses it into a query.Query struct. It may fail. -func Parse(sqls string) (query.Query, error) { - qs, err := ParseMany([]string{sqls}) - if len(qs) == 0 { - return query.Query{}, err - } - return qs[0], err +func Parse(sql string) (query.Query, error) { + sql = strings.TrimSpace(sql) + return (&parser{ + sql: sql, + sqlUpper: strings.ToUpper(sql), + step: stepType, + }).parse() } // ParseMany takes a string slice representing many SQL queries and parses them into a query.Query struct slice. @@ -22,7 +23,7 @@ func Parse(sqls string) (query.Query, error) { func ParseMany(sqls []string) ([]query.Query, error) { qs := []query.Query{} for _, sql := range sqls { - q, err := parse(sql) + q, err := Parse(sql) if err != nil { return qs, err } @@ -31,11 +32,6 @@ func ParseMany(sqls []string) ([]query.Query, error) { return qs, nil } -func parse(sql string) (query.Query, error) { - sql = strings.TrimSpace(sql) - return (&parser{0, 0, "", sql, strings.ToUpper(sql), stepType, query.Query{}, nil, ""}).parse() -} - type step int const ( From fcfdde1af77a165eccb0fbe3a8e82aa5866b6494 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Mon, 2 Aug 2021 00:35:29 +0500 Subject: [PATCH 08/14] add verbose flag for enable/disable log errors to stderr --- README.md | 102 ++++++++++++++++++++++++------------------------ README.template | 6 +-- sql.go | 22 +++++++++-- sql_test.go | 6 +-- 4 files changed, 75 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 3c5d5d8..2fd56a7 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ import ( ) func main() { - query, err := sqlparser.Parse("SELECT a, b, c FROM 'd' WHERE e = '1' AND f > '2'") + query, err := sqlparser.Parse("SELECT a, b, c FROM 'd' WHERE e = '1' AND f > '2'", false) if err != nil { log.Fatal(err) } @@ -25,7 +25,7 @@ func main() { ### Example: SELECT version() as version ``` -query, err := sqlparser.Parse(`SELECT version() as version`) +query, err := sqlparser.Parse(`SELECT version() as version`, false) query.Query { Type: Select @@ -40,7 +40,7 @@ query.Query { ### Example: SELECT works ``` -query, err := sqlparser.Parse(`SELECT a FROM 'b'`) +query, err := sqlparser.Parse(`SELECT a FROM 'b'`, false) query.Query { Type: Select @@ -55,7 +55,7 @@ query.Query { ### Example: SELECT with alias works ``` -query, err := sqlparser.Parse(`SELECT a AS text FROM 'b'`) +query, err := sqlparser.Parse(`SELECT a AS text FROM 'b'`, false) query.Query { Type: Select @@ -70,7 +70,7 @@ query.Query { ### Example: SELECT with alias works ``` -query, err := sqlparser.Parse(`SELECT version(a) AS version FROM 'b'`) +query, err := sqlparser.Parse(`SELECT version(a) AS version FROM 'b'`, false) query.Query { Type: Select @@ -85,7 +85,7 @@ query.Query { ### Example: SELECT works with lowercase ``` -query, err := sqlparser.Parse(`select a fRoM 'b'`) +query, err := sqlparser.Parse(`select a fRoM 'b'`, false) query.Query { Type: Select @@ -100,7 +100,7 @@ query.Query { ### Example: SELECT many fields works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b'`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b'`, false) query.Query { Type: Select @@ -115,7 +115,7 @@ query.Query { ### Example: SELECT with WHERE with = works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a = ''`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a = ''`, false) query.Query { Type: Select @@ -137,7 +137,7 @@ query.Query { ### Example: SELECT with WHERE with < works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a < '1'`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a < '1'`, false) query.Query { Type: Select @@ -159,7 +159,7 @@ query.Query { ### Example: SELECT with WHERE with <= works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a <= '1'`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a <= '1'`, false) query.Query { Type: Select @@ -181,7 +181,7 @@ query.Query { ### Example: SELECT with WHERE with > works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a > '1'`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a > '1'`, false) query.Query { Type: Select @@ -203,7 +203,7 @@ query.Query { ### Example: SELECT with WHERE with >= works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a >= '1'`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a >= '1'`, false) query.Query { Type: Select @@ -225,7 +225,7 @@ query.Query { ### Example: SELECT with WHERE with != works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != '1'`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != '1'`, false) query.Query { Type: Select @@ -247,7 +247,7 @@ query.Query { ### Example: SELECT with WHERE with != works (comparing field against another field) ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != b`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != b`, false) query.Query { Type: Select @@ -269,7 +269,7 @@ query.Query { ### Example: SELECT * works ``` -query, err := sqlparser.Parse(`SELECT * FROM 'b'`) +query, err := sqlparser.Parse(`SELECT * FROM 'b'`, false) query.Query { Type: Select @@ -284,7 +284,7 @@ query.Query { ### Example: SELECT a, * works ``` -query, err := sqlparser.Parse(`SELECT a, * FROM 'b'`) +query, err := sqlparser.Parse(`SELECT a, * FROM 'b'`, false) query.Query { Type: Select @@ -299,7 +299,7 @@ query.Query { ### Example: SELECT with WHERE with two conditions using AND works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != '1' AND b = '2'`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != '1' AND b = '2'`, false) query.Query { Type: Select @@ -328,7 +328,7 @@ query.Query { ### Example: UPDATE works ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE a = '1'`) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE a = '1'`, false) query.Query { Type: Update @@ -350,7 +350,7 @@ query.Query { ### Example: UPDATE works with simple quote inside ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello\'world' WHERE a = '1'`) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello\'world' WHERE a = '1'`, false) query.Query { Type: Update @@ -372,7 +372,7 @@ query.Query { ### Example: UPDATE with multiple SETs works ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello', c = 'bye' WHERE a = '1'`) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello', c = 'bye' WHERE a = '1'`, false) query.Query { Type: Update @@ -394,7 +394,7 @@ query.Query { ### Example: UPDATE with multiple SETs and multiple conditions works ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello', c = 'bye' WHERE a = '1' AND b = '789'`) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello', c = 'bye' WHERE a = '1' AND b = '789'`, false) query.Query { Type: Update @@ -423,7 +423,7 @@ query.Query { ### Example: DELETE with WHERE works ``` -query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE b = '1'`) +query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE b = '1'`, false) query.Query { Type: Delete @@ -445,7 +445,7 @@ query.Query { ### Example: INSERT works ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES ('1')`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES ('1')`, false) query.Query { Type: Insert @@ -460,7 +460,7 @@ query.Query { ### Example: INSERT with multiple fields works ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' )`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' )`, false) query.Query { Type: Insert @@ -475,7 +475,7 @@ query.Query { ### Example: INSERT with multiple fields and multiple values works ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' ),('4','5' ,'6' )`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' ),('4','5' ,'6' )`, false) query.Query { Type: Insert @@ -492,7 +492,7 @@ query.Query { ### Example: empty query fails ``` -query, err := sqlparser.Parse(``) +query, err := sqlparser.Parse(``, false) query type cannot be empty ``` @@ -500,7 +500,7 @@ query type cannot be empty ### Example: SELECT without FROM fails ``` -query, err := sqlparser.Parse(`SELECT`) +query, err := sqlparser.Parse(`SELECT`, false) table name cannot be empty ``` @@ -508,7 +508,7 @@ table name cannot be empty ### Example: SELECT without fields fails ``` -query, err := sqlparser.Parse(`SELECT FROM 'a'`) +query, err := sqlparser.Parse(`SELECT FROM 'a'`, false) at SELECT: expected field to SELECT ``` @@ -516,7 +516,7 @@ at SELECT: expected field to SELECT ### Example: SELECT with comma and empty field fails ``` -query, err := sqlparser.Parse(`SELECT b, FROM 'a'`) +query, err := sqlparser.Parse(`SELECT b, FROM 'a'`, false) at SELECT: expected field to SELECT ``` @@ -524,7 +524,7 @@ at SELECT: expected field to SELECT ### Example: SELECT with incomplete alias fails ``` -query, err := sqlparser.Parse(`SELECT a AS`) +query, err := sqlparser.Parse(`SELECT a AS`, false) at SELECT: expected alias (AS) for a ``` @@ -532,7 +532,7 @@ at SELECT: expected alias (AS) for a ### Example: SELECT with empty WHERE fails ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE`, false) at WHERE: empty WHERE clause ``` @@ -540,7 +540,7 @@ at WHERE: empty WHERE clause ### Example: SELECT with WHERE with only operand fails ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a`) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a`, false) at WHERE: condition without operator ``` @@ -548,7 +548,7 @@ at WHERE: condition without operator ### Example: Empty UPDATE fails ``` -query, err := sqlparser.Parse(`UPDATE`) +query, err := sqlparser.Parse(`UPDATE`, false) table name cannot be empty ``` @@ -556,7 +556,7 @@ table name cannot be empty ### Example: Incomplete UPDATE with table name fails ``` -query, err := sqlparser.Parse(`UPDATE 'a'`) +query, err := sqlparser.Parse(`UPDATE 'a'`, false) at WHERE: WHERE clause is mandatory for UPDATE & DELETE ``` @@ -564,7 +564,7 @@ at WHERE: WHERE clause is mandatory for UPDATE & DELETE ### Example: Incomplete UPDATE with table name and SET fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET`) +query, err := sqlparser.Parse(`UPDATE 'a' SET`, false) at WHERE: WHERE clause is mandatory for UPDATE & DELETE ``` @@ -572,7 +572,7 @@ at WHERE: WHERE clause is mandatory for UPDATE & DELETE ### Example: Incomplete UPDATE with table name, SET with a field but no value and WHERE fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b WHERE`) +query, err := sqlparser.Parse(`UPDATE 'a' SET b WHERE`, false) at UPDATE: expected '=' ``` @@ -580,7 +580,7 @@ at UPDATE: expected '=' ### Example: Incomplete UPDATE with table name, SET with a field and = but no value and WHERE fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = WHERE`) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = WHERE`, false) at UPDATE: expected quoted value ``` @@ -588,7 +588,7 @@ at UPDATE: expected quoted value ### Example: Incomplete UPDATE due to no WHERE clause fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE`) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE`, false) at WHERE: empty WHERE clause ``` @@ -596,7 +596,7 @@ at WHERE: empty WHERE clause ### Example: Incomplete UPDATE due incomplete WHERE clause fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE a`) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE a`, false) at WHERE: condition without operator ``` @@ -604,7 +604,7 @@ at WHERE: condition without operator ### Example: Empty DELETE fails ``` -query, err := sqlparser.Parse(`DELETE FROM`) +query, err := sqlparser.Parse(`DELETE FROM`, false) table name cannot be empty ``` @@ -612,7 +612,7 @@ table name cannot be empty ### Example: DELETE without WHERE fails ``` -query, err := sqlparser.Parse(`DELETE FROM 'a'`) +query, err := sqlparser.Parse(`DELETE FROM 'a'`, false) at WHERE: WHERE clause is mandatory for UPDATE & DELETE ``` @@ -620,7 +620,7 @@ at WHERE: WHERE clause is mandatory for UPDATE & DELETE ### Example: DELETE with empty WHERE fails ``` -query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE`) +query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE`, false) at WHERE: empty WHERE clause ``` @@ -628,7 +628,7 @@ at WHERE: empty WHERE clause ### Example: DELETE with WHERE with field but no operator fails ``` -query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE b`) +query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE b`, false) at WHERE: condition without operator ``` @@ -636,7 +636,7 @@ at WHERE: condition without operator ### Example: Empty INSERT fails ``` -query, err := sqlparser.Parse(`INSERT INTO`) +query, err := sqlparser.Parse(`INSERT INTO`, false) table name cannot be empty ``` @@ -644,7 +644,7 @@ table name cannot be empty ### Example: INSERT with no rows to insert fails ``` -query, err := sqlparser.Parse(`INSERT INTO 'a'`) +query, err := sqlparser.Parse(`INSERT INTO 'a'`, false) at INSERT INTO: need at least one row to insert ``` @@ -652,7 +652,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete value section fails ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (`, false) at INSERT INTO: need at least one row to insert ``` @@ -660,7 +660,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete value section fails #2 ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b`, false) at INSERT INTO: need at least one row to insert ``` @@ -668,7 +668,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete value section fails #3 ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b)`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b)`, false) at INSERT INTO: need at least one row to insert ``` @@ -676,7 +676,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete value section fails #4 ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES`, false) at INSERT INTO: need at least one row to insert ``` @@ -684,7 +684,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete row fails ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES (`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES (`, false) at INSERT INTO: value count doesn't match field count ``` @@ -692,7 +692,7 @@ at INSERT INTO: value count doesn't match field count ### Example: INSERT * fails ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (*) VALUES ('1')`) +query, err := sqlparser.Parse(`INSERT INTO 'a' (*) VALUES ('1')`, false) at INSERT INTO: expected at least one field to insert ``` diff --git a/README.template b/README.template index 0e35490..4297ebd 100644 --- a/README.template +++ b/README.template @@ -15,7 +15,7 @@ import ( ) func main() { - query, err := sqlparser.Parse("SELECT a, b, c FROM 'd' WHERE e = '1' AND f > '2'") + query, err := sqlparser.Parse("SELECT a, b, c FROM 'd' WHERE e = '1' AND f > '2'", false) if err != nil { log.Fatal(err) } @@ -27,7 +27,7 @@ func main() { ### Example: {{.Name}} ``` -query, err := sqlparser.Parse(`{{.SQL}}`) +query, err := sqlparser.Parse(`{{.SQL}}`, false) query.Query { Type: {{index $types .Expected.Type}} @@ -51,7 +51,7 @@ query.Query { ### Example: {{.Name}} ``` -query, err := sqlparser.Parse(`{{.SQL}}`) +query, err := sqlparser.Parse(`{{.SQL}}`, false) {{.Err}} ``` diff --git a/sql.go b/sql.go index a553302..d33e018 100644 --- a/sql.go +++ b/sql.go @@ -9,21 +9,22 @@ import ( ) // Parse takes a string representing a SQL query and parses it into a query.Query struct. It may fail. -func Parse(sql string) (query.Query, error) { +func Parse(sql string, verbose bool) (query.Query, error) { sql = strings.TrimSpace(sql) return (&parser{ sql: sql, sqlUpper: strings.ToUpper(sql), step: stepType, + verbose: verbose, }).parse() } // ParseMany takes a string slice representing many SQL queries and parses them into a query.Query struct slice. // It may fail. If it fails, it will stop at the first failure. -func ParseMany(sqls []string) ([]query.Query, error) { +func ParseMany(sqls []string, verbose bool) ([]query.Query, error) { qs := []query.Query{} for _, sql := range sqls { - q, err := Parse(sql) + q, err := Parse(sql, verbose) if err != nil { return qs, err } @@ -71,7 +72,9 @@ type parser struct { sqlUpper string step step query query.Query + verbose bool err error + errArr [3]string nextUpdateField string } @@ -503,7 +506,7 @@ func (p *parser) validate() error { } func (p *parser) logError() { - if p.err == nil { + if p.err == nil || !p.verbose { return } fmt.Println(p.sql) @@ -511,6 +514,17 @@ func (p *parser) logError() { fmt.Println(p.err) } +func (p *parser) FormatError() (error, [3]string) { + if p.err == nil { + return nil, p.errArr + } + p.errArr[0] = p.sql + p.errArr[1] = strings.Repeat(" ", p.i) + "^" + p.errArr[1] = p.err.Error() + + return p.err, p.errArr +} + var regexIdentifier = regexp.MustCompile("[a-zA-Z_][a-zA-Z_0-9]*") func isIdentifier(s string) bool { diff --git a/sql_test.go b/sql_test.go index 70e2655..c8b18c4 100644 --- a/sql_test.go +++ b/sql_test.go @@ -451,7 +451,7 @@ func TestSQL(t *testing.T) { output := output{Types: query.TypeString, Operators: query.OperatorString} for _, tc := range ts { t.Run(tc.Name, func(t *testing.T) { - actual, err := ParseMany([]string{tc.SQL}) + actual, err := ParseMany([]string{tc.SQL}, true) if tc.Err != nil && err == nil { t.Errorf("Error should have been %v", tc.Err) } @@ -477,7 +477,7 @@ func TestSQL(t *testing.T) { func BenchmarkSQLSelect(b *testing.B) { sql := "SELECT a AS text FROM 'b' WHERE c = 'c' AND d = 'd'" for i := 0; i < b.N; i++ { - q, err := Parse(sql) + q, err := Parse(sql, false) if err != nil { b.Errorf("Error should have been %v: %v", err, q) } @@ -487,7 +487,7 @@ func BenchmarkSQLSelect(b *testing.B) { func BenchmarkSQLInsert(b *testing.B) { sql := "INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' )" for i := 0; i < b.N; i++ { - q, err := Parse(sql) + q, err := Parse(sql, false) if err != nil { b.Errorf("Error should have been %v: %v", err, q) } From 01b43545e0b2ca3845214fd055e098631f5889f4 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Mon, 2 Aug 2021 01:21:51 +0500 Subject: [PATCH 09/14] optimize perfomance #4: remove regexp from isIdentifier --- sql.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/sql.go b/sql.go index d33e018..5e0feb8 100644 --- a/sql.go +++ b/sql.go @@ -2,7 +2,6 @@ package sqlparser import ( "fmt" - "regexp" "strings" "github.com/marianogappa/sqlparser/query" @@ -525,16 +524,34 @@ func (p *parser) FormatError() (error, [3]string) { return p.err, p.errArr } -var regexIdentifier = regexp.MustCompile("[a-zA-Z_][a-zA-Z_0-9]*") +//var regexIdentifier = regexp.MustCompile("[a-zA-Z_][a-zA-Z_0-9]*") func isIdentifier(s string) bool { + if len(s) == 0 { + return false + } u := strings.ToUpper(s) for _, rw := range reservedWords { if u == rw { return false } } - return regexIdentifier.MatchString(s) + //return regexIdentifier.MatchString(s) + if (s[0] >= 'a' && s[0] <= 'z') || + (s[0] >= 'A' && s[0] <= 'Z') || + s[0] == '_' { + // for i := 1; i < len(s); i++ { + // isIdentifierSymbol := (s[i] >= 'a' && s[i] <= 'z') || + // (s[i] >= 'A' && s[i] <= 'Z') || + // (s[i] >= '0' && s[i] <= '9') || + // s[i] == '_' + // if !isIdentifierSymbol { + // return false + // } + // } + return true + } + return false } func isIdentifierOrAsterisk(s string) bool { From 7bd34a705e3160772133a83918d5b3dd82320b9c Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Mon, 2 Aug 2021 13:34:28 +0500 Subject: [PATCH 10/14] some refactorfor parser.FormatError() --- sql.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sql.go b/sql.go index 5e0feb8..a92666e 100644 --- a/sql.go +++ b/sql.go @@ -364,6 +364,14 @@ func (p *parser) doParse() (query.Query, error) { } } +func (p *parser) peekCurrent(upper bool) string { + if upper { + return p.sqlUpper[p.i : p.i+p.len] + } else { + return p.sql[p.i : p.i+p.len] + } +} + func (p *parser) peek(upper bool) string { p.peeked, p.len = p.peekWithLength(upper) return p.peeked @@ -513,15 +521,15 @@ func (p *parser) logError() { fmt.Println(p.err) } -func (p *parser) FormatError() (error, [3]string) { +func (p *parser) FormatError() ([3]string, error) { if p.err == nil { - return nil, p.errArr + return p.errArr, nil } p.errArr[0] = p.sql p.errArr[1] = strings.Repeat(" ", p.i) + "^" p.errArr[1] = p.err.Error() - return p.err, p.errArr + return p.errArr, p.err } //var regexIdentifier = regexp.MustCompile("[a-zA-Z_][a-zA-Z_0-9]*") From 74b2decdabf8bda177e14ae66117f157ebffbd98 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Mon, 2 Aug 2021 13:43:02 +0500 Subject: [PATCH 11/14] rename --- README.md | 9 ++++++--- README.template | 9 ++++++--- go.mod | 2 +- sql.go | 2 +- sql_test.go | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2fd56a7..b078f4c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ -# sqlparser - meant for querying csv files -[![Build Status](https://img.shields.io/travis/marianogappa/sqlparser.svg)](https://travis-ci.org/marianogappa/sqlparser) [![Coverage Status](https://coveralls.io/repos/github/marianogappa/sqlparser/badge.svg?branch=master)](https://coveralls.io/github/MarianoGappa/sqlparser?branch=master) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/marianogappa/sqlparser/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/marianogappa/sqlparser?style=flat-square)](https://goreportcard.com/report/github.com/marianogappa/sqlparser) [![GoDoc](https://godoc.org/github.com/marianogappa/sqlparser?status.svg)](https://godoc.org/github.com/marianogappa/sqlparser) +# sqlparser - simple SQL parser + +Based on https://github.com/marianogappa/sqlparser + +Documentation on https://godoc.org/github.com/msaf1980/sqlparser ### Usage ``` @@ -9,7 +12,7 @@ import ( "fmt" "log" - "github.com/marianogappa/sqlparser" + "github.com/msaf1980/sqlparser" ) func main() { diff --git a/README.template b/README.template index 4297ebd..33ed52c 100644 --- a/README.template +++ b/README.template @@ -1,7 +1,10 @@ {{- $types := .Types -}} {{- $operators := .Operators -}} -# sqlparser - meant for querying csv files -[![Build Status](https://img.shields.io/travis/marianogappa/sqlparser.svg)](https://travis-ci.org/marianogappa/sqlparser) [![Coverage Status](https://coveralls.io/repos/github/marianogappa/sqlparser/badge.svg?branch=master)](https://coveralls.io/github/MarianoGappa/sqlparser?branch=master) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/marianogappa/sqlparser/master/LICENSE) [![Go Report Card](https://goreportcard.com/badge/github.com/marianogappa/sqlparser?style=flat-square)](https://goreportcard.com/report/github.com/marianogappa/sqlparser) [![GoDoc](https://godoc.org/github.com/marianogappa/sqlparser?status.svg)](https://godoc.org/github.com/marianogappa/sqlparser) +# sqlparser - simple SQL parser + +Based on https://github.com/marianogappa/sqlparser + +Documentation on https://godoc.org/github.com/msaf1980/sqlparser ### Usage ``` @@ -11,7 +14,7 @@ import ( "fmt" "log" - "github.com/marianogappa/sqlparser" + "github.com/msaf1980/sqlparser" ) func main() { diff --git a/go.mod b/go.mod index b6e0dd3..f6e2d96 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/marianogappa/sqlparser +module github.com/msaf1980/sqlparser go 1.14 diff --git a/sql.go b/sql.go index a92666e..0cb7b10 100644 --- a/sql.go +++ b/sql.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/marianogappa/sqlparser/query" + "github.com/msaf1980/sqlparser/query" ) // Parse takes a string representing a SQL query and parses it into a query.Query struct. It may fail. diff --git a/sql_test.go b/sql_test.go index c8b18c4..eaad94c 100644 --- a/sql_test.go +++ b/sql_test.go @@ -8,7 +8,7 @@ import ( "testing" "text/template" - "github.com/marianogappa/sqlparser/query" + "github.com/msaf1980/sqlparser/query" "github.com/stretchr/testify/require" ) From 8d8a9661bdb121585cbd998255b3f224212943d1 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Mon, 2 Aug 2021 13:53:17 +0500 Subject: [PATCH 12/14] tests: add ci tests --- .github/workflows/codeql.yml | 64 ++++++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 34 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..1c2395b --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,64 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [master] + pull_request: + # The branches below must be a subset of the branches above + branches: [master] + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['go'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..ef4ac8b --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,34 @@ +name: Tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + tests: + name: Test code + runs-on: ubuntu-latest + strategy: + matrix: + go: + - ^1.14 + - ^1.15 + - ^1.16 + - ^1 + steps: + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + with: + fetch-depth: 200 + + - name: Test + run: go test ./... -v From b363c596b67608311d5cad0ba4a74824af9fa8c7 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Mon, 2 Aug 2021 17:59:47 +0500 Subject: [PATCH 13/14] sqlparser: operand type refactor --- README.md | 162 ++++++++++++++--------------- README.template | 10 +- query/query.go | 13 ++- sql.go | 271 +++++++++++++++++++++++++++--------------------- sql_test.go | 117 +++++++++++++++++---- 5 files changed, 350 insertions(+), 223 deletions(-) diff --git a/README.md b/README.md index b078f4c..b8e8851 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ import ( ) func main() { - query, err := sqlparser.Parse("SELECT a, b, c FROM 'd' WHERE e = '1' AND f > '2'", false) + query, err := sqlparser.Parse("SELECT a, b, c FROM 'd' WHERE e = '1' AND f > '2'") if err != nil { log.Fatal(err) } @@ -28,7 +28,7 @@ func main() { ### Example: SELECT version() as version ``` -query, err := sqlparser.Parse(`SELECT version() as version`, false) +query, err := sqlparser.Parse(`SELECT version() as version`) query.Query { Type: Select @@ -43,7 +43,7 @@ query.Query { ### Example: SELECT works ``` -query, err := sqlparser.Parse(`SELECT a FROM 'b'`, false) +query, err := sqlparser.Parse(`SELECT a FROM 'b'`) query.Query { Type: Select @@ -58,7 +58,7 @@ query.Query { ### Example: SELECT with alias works ``` -query, err := sqlparser.Parse(`SELECT a AS text FROM 'b'`, false) +query, err := sqlparser.Parse(`SELECT a AS text FROM 'b'`) query.Query { Type: Select @@ -73,7 +73,7 @@ query.Query { ### Example: SELECT with alias works ``` -query, err := sqlparser.Parse(`SELECT version(a) AS version FROM 'b'`, false) +query, err := sqlparser.Parse(`SELECT version(a) AS version FROM 'b'`) query.Query { Type: Select @@ -88,7 +88,7 @@ query.Query { ### Example: SELECT works with lowercase ``` -query, err := sqlparser.Parse(`select a fRoM 'b'`, false) +query, err := sqlparser.Parse(`select a fRoM 'b'`) query.Query { Type: Select @@ -103,7 +103,7 @@ query.Query { ### Example: SELECT many fields works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b'`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b'`) query.Query { Type: Select @@ -118,7 +118,7 @@ query.Query { ### Example: SELECT with WHERE with = works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a = ''`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a = ''`) query.Query { Type: Select @@ -126,10 +126,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Eq, Operand2: , - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[] Inserts: [] @@ -140,7 +140,7 @@ query.Query { ### Example: SELECT with WHERE with < works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a < '1'`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a < '1'`) query.Query { Type: Select @@ -148,10 +148,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Lt, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[] Inserts: [] @@ -162,7 +162,7 @@ query.Query { ### Example: SELECT with WHERE with <= works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a <= '1'`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a <= '1'`) query.Query { Type: Select @@ -170,10 +170,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Lte, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[] Inserts: [] @@ -184,7 +184,7 @@ query.Query { ### Example: SELECT with WHERE with > works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a > '1'`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a > '1'`) query.Query { Type: Select @@ -192,10 +192,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Gt, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[] Inserts: [] @@ -206,7 +206,7 @@ query.Query { ### Example: SELECT with WHERE with >= works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a >= '1'`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a >= '1'`) query.Query { Type: Select @@ -214,10 +214,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Gte, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[] Inserts: [] @@ -228,7 +228,7 @@ query.Query { ### Example: SELECT with WHERE with != works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != '1'`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != '1'`) query.Query { Type: Select @@ -236,10 +236,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Ne, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[] Inserts: [] @@ -250,7 +250,7 @@ query.Query { ### Example: SELECT with WHERE with != works (comparing field against another field) ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != b`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != b`) query.Query { Type: Select @@ -258,10 +258,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Ne, Operand2: b, - Operand2IsField: true, + Operand2Type: 1, }] Updates: map[] Inserts: [] @@ -272,7 +272,7 @@ query.Query { ### Example: SELECT * works ``` -query, err := sqlparser.Parse(`SELECT * FROM 'b'`, false) +query, err := sqlparser.Parse(`SELECT * FROM 'b'`) query.Query { Type: Select @@ -287,7 +287,7 @@ query.Query { ### Example: SELECT a, * works ``` -query, err := sqlparser.Parse(`SELECT a, * FROM 'b'`, false) +query, err := sqlparser.Parse(`SELECT a, * FROM 'b'`) query.Query { Type: Select @@ -302,7 +302,7 @@ query.Query { ### Example: SELECT with WHERE with two conditions using AND works ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != '1' AND b = '2'`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a != '1' AND b = '2'`) query.Query { Type: Select @@ -310,17 +310,17 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Ne, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, } { Operand1: b, - Operand1IsField: true, + Operand1Type: 1, Operator: Eq, Operand2: 2, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[] Inserts: [] @@ -331,7 +331,7 @@ query.Query { ### Example: UPDATE works ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE a = '1'`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE a = '1'`) query.Query { Type: Update @@ -339,10 +339,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Eq, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[b:hello] Inserts: [] @@ -353,7 +353,7 @@ query.Query { ### Example: UPDATE works with simple quote inside ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello\'world' WHERE a = '1'`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello\'world' WHERE a = '1'`) query.Query { Type: Update @@ -361,10 +361,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Eq, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[b:hello\'world] Inserts: [] @@ -375,7 +375,7 @@ query.Query { ### Example: UPDATE with multiple SETs works ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello', c = 'bye' WHERE a = '1'`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello', c = 'bye' WHERE a = '1'`) query.Query { Type: Update @@ -383,10 +383,10 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Eq, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[b:hello c:bye] Inserts: [] @@ -397,7 +397,7 @@ query.Query { ### Example: UPDATE with multiple SETs and multiple conditions works ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello', c = 'bye' WHERE a = '1' AND b = '789'`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello', c = 'bye' WHERE a = '1' AND b = '789'`) query.Query { Type: Update @@ -405,17 +405,17 @@ query.Query { Conditions: [ { Operand1: a, - Operand1IsField: true, + Operand1Type: 1, Operator: Eq, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, } { Operand1: b, - Operand1IsField: true, + Operand1Type: 1, Operator: Eq, Operand2: 789, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[b:hello c:bye] Inserts: [] @@ -426,7 +426,7 @@ query.Query { ### Example: DELETE with WHERE works ``` -query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE b = '1'`, false) +query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE b = '1'`) query.Query { Type: Delete @@ -434,10 +434,10 @@ query.Query { Conditions: [ { Operand1: b, - Operand1IsField: true, + Operand1Type: 1, Operator: Eq, Operand2: 1, - Operand2IsField: false, + Operand2Type: 2, }] Updates: map[] Inserts: [] @@ -448,7 +448,7 @@ query.Query { ### Example: INSERT works ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES ('1')`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES ('1')`) query.Query { Type: Insert @@ -463,7 +463,7 @@ query.Query { ### Example: INSERT with multiple fields works ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' )`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' )`) query.Query { Type: Insert @@ -478,7 +478,7 @@ query.Query { ### Example: INSERT with multiple fields and multiple values works ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' ),('4','5' ,'6' )`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' ),('4','5' ,'6' )`) query.Query { Type: Insert @@ -495,7 +495,7 @@ query.Query { ### Example: empty query fails ``` -query, err := sqlparser.Parse(``, false) +query, err := sqlparser.Parse(``) query type cannot be empty ``` @@ -503,7 +503,7 @@ query type cannot be empty ### Example: SELECT without FROM fails ``` -query, err := sqlparser.Parse(`SELECT`, false) +query, err := sqlparser.Parse(`SELECT`) table name cannot be empty ``` @@ -511,7 +511,7 @@ table name cannot be empty ### Example: SELECT without fields fails ``` -query, err := sqlparser.Parse(`SELECT FROM 'a'`, false) +query, err := sqlparser.Parse(`SELECT FROM 'a'`) at SELECT: expected field to SELECT ``` @@ -519,7 +519,7 @@ at SELECT: expected field to SELECT ### Example: SELECT with comma and empty field fails ``` -query, err := sqlparser.Parse(`SELECT b, FROM 'a'`, false) +query, err := sqlparser.Parse(`SELECT b, FROM 'a'`) at SELECT: expected field to SELECT ``` @@ -527,7 +527,7 @@ at SELECT: expected field to SELECT ### Example: SELECT with incomplete alias fails ``` -query, err := sqlparser.Parse(`SELECT a AS`, false) +query, err := sqlparser.Parse(`SELECT a AS`) at SELECT: expected alias (AS) for a ``` @@ -535,7 +535,7 @@ at SELECT: expected alias (AS) for a ### Example: SELECT with empty WHERE fails ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE`) at WHERE: empty WHERE clause ``` @@ -543,7 +543,7 @@ at WHERE: empty WHERE clause ### Example: SELECT with WHERE with only operand fails ``` -query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a`, false) +query, err := sqlparser.Parse(`SELECT a, c, d FROM 'b' WHERE a`) at WHERE: condition without operator ``` @@ -551,7 +551,7 @@ at WHERE: condition without operator ### Example: Empty UPDATE fails ``` -query, err := sqlparser.Parse(`UPDATE`, false) +query, err := sqlparser.Parse(`UPDATE`) table name cannot be empty ``` @@ -559,7 +559,7 @@ table name cannot be empty ### Example: Incomplete UPDATE with table name fails ``` -query, err := sqlparser.Parse(`UPDATE 'a'`, false) +query, err := sqlparser.Parse(`UPDATE 'a'`) at WHERE: WHERE clause is mandatory for UPDATE & DELETE ``` @@ -567,7 +567,7 @@ at WHERE: WHERE clause is mandatory for UPDATE & DELETE ### Example: Incomplete UPDATE with table name and SET fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET`) at WHERE: WHERE clause is mandatory for UPDATE & DELETE ``` @@ -575,7 +575,7 @@ at WHERE: WHERE clause is mandatory for UPDATE & DELETE ### Example: Incomplete UPDATE with table name, SET with a field but no value and WHERE fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b WHERE`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET b WHERE`) at UPDATE: expected '=' ``` @@ -583,7 +583,7 @@ at UPDATE: expected '=' ### Example: Incomplete UPDATE with table name, SET with a field and = but no value and WHERE fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = WHERE`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = WHERE`) at UPDATE: expected quoted value ``` @@ -591,7 +591,7 @@ at UPDATE: expected quoted value ### Example: Incomplete UPDATE due to no WHERE clause fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE`) at WHERE: empty WHERE clause ``` @@ -599,7 +599,7 @@ at WHERE: empty WHERE clause ### Example: Incomplete UPDATE due incomplete WHERE clause fails ``` -query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE a`, false) +query, err := sqlparser.Parse(`UPDATE 'a' SET b = 'hello' WHERE a`) at WHERE: condition without operator ``` @@ -607,7 +607,7 @@ at WHERE: condition without operator ### Example: Empty DELETE fails ``` -query, err := sqlparser.Parse(`DELETE FROM`, false) +query, err := sqlparser.Parse(`DELETE FROM`) table name cannot be empty ``` @@ -615,7 +615,7 @@ table name cannot be empty ### Example: DELETE without WHERE fails ``` -query, err := sqlparser.Parse(`DELETE FROM 'a'`, false) +query, err := sqlparser.Parse(`DELETE FROM 'a'`) at WHERE: WHERE clause is mandatory for UPDATE & DELETE ``` @@ -623,7 +623,7 @@ at WHERE: WHERE clause is mandatory for UPDATE & DELETE ### Example: DELETE with empty WHERE fails ``` -query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE`, false) +query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE`) at WHERE: empty WHERE clause ``` @@ -631,7 +631,7 @@ at WHERE: empty WHERE clause ### Example: DELETE with WHERE with field but no operator fails ``` -query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE b`, false) +query, err := sqlparser.Parse(`DELETE FROM 'a' WHERE b`) at WHERE: condition without operator ``` @@ -639,7 +639,7 @@ at WHERE: condition without operator ### Example: Empty INSERT fails ``` -query, err := sqlparser.Parse(`INSERT INTO`, false) +query, err := sqlparser.Parse(`INSERT INTO`) table name cannot be empty ``` @@ -647,7 +647,7 @@ table name cannot be empty ### Example: INSERT with no rows to insert fails ``` -query, err := sqlparser.Parse(`INSERT INTO 'a'`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a'`) at INSERT INTO: need at least one row to insert ``` @@ -655,7 +655,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete value section fails ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (`) at INSERT INTO: need at least one row to insert ``` @@ -663,7 +663,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete value section fails #2 ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b`) at INSERT INTO: need at least one row to insert ``` @@ -671,7 +671,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete value section fails #3 ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b)`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b)`) at INSERT INTO: need at least one row to insert ``` @@ -679,7 +679,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete value section fails #4 ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES`) at INSERT INTO: need at least one row to insert ``` @@ -687,7 +687,7 @@ at INSERT INTO: need at least one row to insert ### Example: INSERT with incomplete row fails ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES (`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (b) VALUES (`) at INSERT INTO: value count doesn't match field count ``` @@ -695,7 +695,7 @@ at INSERT INTO: value count doesn't match field count ### Example: INSERT * fails ``` -query, err := sqlparser.Parse(`INSERT INTO 'a' (*) VALUES ('1')`, false) +query, err := sqlparser.Parse(`INSERT INTO 'a' (*) VALUES ('1')`) at INSERT INTO: expected at least one field to insert ``` diff --git a/README.template b/README.template index 33ed52c..95c295b 100644 --- a/README.template +++ b/README.template @@ -18,7 +18,7 @@ import ( ) func main() { - query, err := sqlparser.Parse("SELECT a, b, c FROM 'd' WHERE e = '1' AND f > '2'", false) + query, err := sqlparser.Parse("SELECT a, b, c FROM 'd' WHERE e = '1' AND f > '2'") if err != nil { log.Fatal(err) } @@ -30,7 +30,7 @@ func main() { ### Example: {{.Name}} ``` -query, err := sqlparser.Parse(`{{.SQL}}`, false) +query, err := sqlparser.Parse(`{{.SQL}}`) query.Query { Type: {{index $types .Expected.Type}} @@ -38,10 +38,10 @@ query.Query { Conditions: [{{range .Expected.Conditions}} { Operand1: {{.Operand1}}, - Operand1IsField: {{.Operand1IsField}}, + Operand1Type: {{.Operand1Type}}, Operator: {{index $operators .Operator}}, Operand2: {{.Operand2}}, - Operand2IsField: {{.Operand2IsField}}, + Operand2Type: {{.Operand2Type}}, }{{end -}}] Updates: {{.Expected.Updates}} Inserts: {{.Expected.Inserts}} @@ -54,7 +54,7 @@ query.Query { ### Example: {{.Name}} ``` -query, err := sqlparser.Parse(`{{.SQL}}`, false) +query, err := sqlparser.Parse(`{{.SQL}}`) {{.Err}} ``` diff --git a/query/query.go b/query/query.go index ed9cd46..3291422 100644 --- a/query/query.go +++ b/query/query.go @@ -67,16 +67,25 @@ var OperatorString = []string{ "Lte", } +type OperandType int + +const ( + OpUnknown OperandType = iota + OpField + OpQuoted + OpUnquoted +) + // Condition is a single boolean condition in a WHERE clause type Condition struct { // Operand1 is the left hand side operand Operand1 string // Operand1IsField determines if Operand1 is a literal or a field name - Operand1IsField bool + Operand1Type OperandType // Operator is e.g. "=", ">" Operator Operator // Operand1 is the right hand side operand Operand2 string // Operand2IsField determines if Operand2 is a literal or a field name - Operand2IsField bool + Operand2Type OperandType } diff --git a/sql.go b/sql.go index 0cb7b10..e8f21b4 100644 --- a/sql.go +++ b/sql.go @@ -2,28 +2,61 @@ package sqlparser import ( "fmt" + "io" "strings" "github.com/msaf1980/sqlparser/query" ) +type ErrorWithPos struct { + msg string + pos int +} + +func newError(pos int, msg string) *ErrorWithPos { + return &ErrorWithPos{ + msg: msg, + pos: pos, + } +} + +func newErrorf(pos int, format string, a ...interface{}) *ErrorWithPos { + return &ErrorWithPos{ + msg: fmt.Sprintf(format, a...), + pos: pos, + } +} + +func (e *ErrorWithPos) Error() string { + return e.msg +} + +func (e *ErrorWithPos) Pos() int { + return e.pos +} + +func (e *ErrorWithPos) PrintPosError(sql string, w io.Writer) { + fmt.Fprintln(w, sql) + fmt.Fprintln(w, strings.Repeat(" ", e.pos)+"^") + fmt.Println(e.msg) +} + // Parse takes a string representing a SQL query and parses it into a query.Query struct. It may fail. -func Parse(sql string, verbose bool) (query.Query, error) { +func Parse(sql string) (query.Query, error) { sql = strings.TrimSpace(sql) return (&parser{ sql: sql, sqlUpper: strings.ToUpper(sql), step: stepType, - verbose: verbose, }).parse() } // ParseMany takes a string slice representing many SQL queries and parses them into a query.Query struct slice. // It may fail. If it fails, it will stop at the first failure. -func ParseMany(sqls []string, verbose bool) ([]query.Query, error) { +func ParseMany(sqls []string) ([]query.Query, error) { qs := []query.Query{} for _, sql := range sqls { - q, err := Parse(sql, verbose) + q, err := Parse(sql) if err != nil { return qs, err } @@ -71,9 +104,7 @@ type parser struct { sqlUpper string step step query query.Query - verbose bool err error - errArr [3]string nextUpdateField string } @@ -83,7 +114,6 @@ func (p *parser) parse() (query.Query, error) { if p.err == nil { p.err = p.validate() } - p.logError() return q, p.err } @@ -110,13 +140,13 @@ func (p *parser) doParse() (query.Query, error) { p.query.Type = query.Delete p.step = stepDeleteFromTable default: - return p.query, fmt.Errorf("invalid query type") + return p.query, newError(p.i, "invalid query type") } p.pop() case stepSelectField: identifier := p.peek(false) if !isIdentifierOrAsterisk(identifier) { - return p.query, fmt.Errorf("at SELECT: expected field to SELECT") + return p.query, newError(p.i, "at SELECT: expected field to SELECT") } p.query.Fields = append(p.query.Fields, identifier) p.pop() @@ -126,7 +156,7 @@ func (p *parser) doParse() (query.Query, error) { p.pop() alias := p.peek(false) if !isIdentifierOrAsterisk(alias) { - return p.query, fmt.Errorf("at SELECT: expected alias (AS) for %s", identifier) + return p.query, newErrorf(p.i, "at SELECT: expected alias (AS) for %s", identifier) } p.query.Aliases = append(p.query.Aliases, alias) p.pop() @@ -142,21 +172,21 @@ func (p *parser) doParse() (query.Query, error) { case stepSelectComma: commaRWord := p.peek(false) if commaRWord != "," { - return p.query, fmt.Errorf("at SELECT: expected comma or FROM") + return p.query, newError(p.i, "at SELECT: expected comma or FROM") } p.pop() p.step = stepSelectField case stepSelectFrom: fromRWord := p.peek(true) if fromRWord != "FROM" { - return p.query, fmt.Errorf("at SELECT: expected FROM") + return p.query, newError(p.i, "at SELECT: expected FROM") } p.pop() p.step = stepSelectFromTable case stepSelectFromTable: tableName := p.peek(false) if len(tableName) == 0 { - return p.query, fmt.Errorf("at SELECT: expected quoted table name") + return p.query, newError(p.i, "at SELECT: expected quoted table name") } p.query.TableName = tableName p.pop() @@ -164,7 +194,7 @@ func (p *parser) doParse() (query.Query, error) { case stepInsertTable: tableName := p.peek(false) if len(tableName) == 0 { - return p.query, fmt.Errorf("at INSERT INTO: expected quoted table name") + return p.query, newError(p.i, "at INSERT INTO: expected quoted table name") } p.query.TableName = tableName p.pop() @@ -172,7 +202,7 @@ func (p *parser) doParse() (query.Query, error) { case stepDeleteFromTable: tableName := p.peek(false) if len(tableName) == 0 { - return p.query, fmt.Errorf("at DELETE FROM: expected quoted table name") + return p.query, newError(p.i, "at DELETE FROM: expected quoted table name") } p.query.TableName = tableName p.pop() @@ -180,7 +210,7 @@ func (p *parser) doParse() (query.Query, error) { case stepUpdateTable: tableName := p.peek(false) if len(tableName) == 0 { - return p.query, fmt.Errorf("at UPDATE: expected quoted table name") + return p.query, newError(p.i, "at UPDATE: expected quoted table name") } p.query.TableName = tableName p.pop() @@ -188,14 +218,14 @@ func (p *parser) doParse() (query.Query, error) { case stepUpdateSet: setRWord := p.peek(true) if setRWord != "SET" { - return p.query, fmt.Errorf("at UPDATE: expected 'SET'") + return p.query, newError(p.i, "at UPDATE: expected 'SET'") } p.pop() p.step = stepUpdateField case stepUpdateField: identifier := p.peek(false) if !isIdentifier(identifier) { - return p.query, fmt.Errorf("at UPDATE: expected at least one field to update") + return p.query, newError(p.i, "at UPDATE: expected at least one field to update") } p.nextUpdateField = identifier p.pop() @@ -203,14 +233,14 @@ func (p *parser) doParse() (query.Query, error) { case stepUpdateEquals: equalsRWord := p.peek(false) if equalsRWord != "=" { - return p.query, fmt.Errorf("at UPDATE: expected '='") + return p.query, newError(p.i, "at UPDATE: expected '='") } p.pop() p.step = stepUpdateValue case stepUpdateValue: quotedValue := p.peekQuotedString(false) if p.len == 0 { - return p.query, fmt.Errorf("at UPDATE: expected quoted value") + return p.query, newError(p.i, "at UPDATE: expected quoted value") } p.query.Updates[p.nextUpdateField] = quotedValue p.nextUpdateField = "" @@ -224,82 +254,31 @@ func (p *parser) doParse() (query.Query, error) { case stepUpdateComma: commaRWord := p.peek(false) if commaRWord != "," { - return p.query, fmt.Errorf("at UPDATE: expected ','") + return p.query, newError(p.i, "at UPDATE: expected ','") } p.pop() p.step = stepUpdateField case stepWhere: whereRWord := p.peek(true) if whereRWord != "WHERE" { - return p.query, fmt.Errorf("expected WHERE") + return p.query, newError(p.i, "expected WHERE") } p.pop() p.step = stepWhereField - case stepWhereField: - identifier := p.peek(false) - if !isIdentifier(identifier) { - return p.query, fmt.Errorf("at WHERE: expected field") - } - p.query.Conditions = append(p.query.Conditions, query.Condition{Operand1: identifier, Operand1IsField: true}) - p.pop() - p.step = stepWhereOperator - case stepWhereOperator: - operator := p.peek(false) - currentCondition := p.query.Conditions[len(p.query.Conditions)-1] - switch operator { - case "=": - currentCondition.Operator = query.Eq - case ">": - currentCondition.Operator = query.Gt - case ">=": - currentCondition.Operator = query.Gte - case "<": - currentCondition.Operator = query.Lt - case "<=": - currentCondition.Operator = query.Lte - case "!=": - currentCondition.Operator = query.Ne - default: - return p.query, fmt.Errorf("at WHERE: unknown operator") - } - p.query.Conditions[len(p.query.Conditions)-1] = currentCondition - p.pop() - p.step = stepWhereValue - case stepWhereValue: - currentCondition := p.query.Conditions[len(p.query.Conditions)-1] - identifier := p.peek(false) - if isIdentifier(identifier) { - currentCondition.Operand2 = identifier - currentCondition.Operand2IsField = true - } else { - quotedValue := p.peekQuotedString(false) - if p.len == 0 { - return p.query, fmt.Errorf("at WHERE: expected quoted value") - } - currentCondition.Operand2 = quotedValue - currentCondition.Operand2IsField = false - } - p.query.Conditions[len(p.query.Conditions)-1] = currentCondition - p.pop() - p.step = stepWhereAnd - case stepWhereAnd: - andRWord := p.peek(true) - if andRWord != "AND" { - return p.query, fmt.Errorf("expected AND") + if ended, err := p.parseWhere(); ended || err != nil { + return p.query, err } - p.pop() - p.step = stepWhereField case stepInsertFieldsOpeningParens: openingParens := p.peek(false) if len(openingParens) != 1 || openingParens != "(" { - return p.query, fmt.Errorf("at INSERT INTO: expected opening parens") + return p.query, newError(p.i, "at INSERT INTO: expected opening parens") } p.pop() p.step = stepInsertFields case stepInsertFields: identifier := p.peek(false) if !isIdentifier(identifier) { - return p.query, fmt.Errorf("at INSERT INTO: expected at least one field to insert") + return p.query, newError(p.i, "at INSERT INTO: expected at least one field to insert") } p.query.Fields = append(p.query.Fields, identifier) p.pop() @@ -307,7 +286,7 @@ func (p *parser) doParse() (query.Query, error) { case stepInsertFieldsCommaOrClosingParens: commaOrClosingParens := p.peek(false) if commaOrClosingParens != "," && commaOrClosingParens != ")" { - return p.query, fmt.Errorf("at INSERT INTO: expected comma or closing parens") + return p.query, newError(p.i, "at INSERT INTO: expected comma or closing parens") } p.pop() if commaOrClosingParens == "," { @@ -318,14 +297,14 @@ func (p *parser) doParse() (query.Query, error) { case stepInsertValuesRWord: valuesRWord := p.peek(true) if valuesRWord != "VALUES" { - return p.query, fmt.Errorf("at INSERT INTO: expected 'VALUES'") + return p.query, newError(p.i, "at INSERT INTO: expected 'VALUES'") } p.pop() p.step = stepInsertValuesOpeningParens case stepInsertValuesOpeningParens: openingParens := p.peek(false) if openingParens != "(" { - return p.query, fmt.Errorf("at INSERT INTO: expected opening parens") + return p.query, newError(p.i, "at INSERT INTO: expected opening parens") } p.query.Inserts = append(p.query.Inserts, []string{}) p.pop() @@ -333,7 +312,7 @@ func (p *parser) doParse() (query.Query, error) { case stepInsertValues: quotedValue := p.peekQuotedString(false) if p.len == 0 { - return p.query, fmt.Errorf("at INSERT INTO: expected quoted value") + return p.query, newError(p.i, "at INSERT INTO: expected quoted value") } p.query.Inserts[len(p.query.Inserts)-1] = append(p.query.Inserts[len(p.query.Inserts)-1], quotedValue) p.pop() @@ -341,7 +320,7 @@ func (p *parser) doParse() (query.Query, error) { case stepInsertValuesCommaOrClosingParens: commaOrClosingParens := p.peek(false) if commaOrClosingParens != "," && commaOrClosingParens != ")" { - return p.query, fmt.Errorf("at INSERT INTO: expected comma or closing parens") + return p.query, newError(p.i, "at INSERT INTO: expected comma or closing parens") } p.pop() if commaOrClosingParens == "," { @@ -350,13 +329,13 @@ func (p *parser) doParse() (query.Query, error) { } currentInsertRow := p.query.Inserts[len(p.query.Inserts)-1] if len(currentInsertRow) < len(p.query.Fields) { - return p.query, fmt.Errorf("at INSERT INTO: value count doesn't match field count") + return p.query, newError(p.i, "at INSERT INTO: value count doesn't match field count") } p.step = stepInsertValuesCommaBeforeOpeningParens case stepInsertValuesCommaBeforeOpeningParens: commaRWord := p.peek(false) if commaRWord != "," { - return p.query, fmt.Errorf("at INSERT INTO: expected comma") + return p.query, newError(p.i, "at INSERT INTO: expected comma") } p.pop() p.step = stepInsertValuesOpeningParens @@ -364,6 +343,86 @@ func (p *parser) doParse() (query.Query, error) { } } +func (p *parser) parseWhere() (bool, error) { + for { + if p.i >= len(p.sql) { + if len(p.query.Conditions) == 0 { + return true, newError(p.i, "at WHERE: empty WHERE clause") + } + // TODO detect closed + + return true, nil + } + switch p.step { + case stepWhereField: + identifier := p.peek(false) + if len(identifier) == 0 { + return false, newError(p.i, "at WHERE: empty WHERE clause") + } else if !isIdentifier(identifier) { + if len(p.query.Conditions) == 0 { + return true, newError(p.i, "at WHERE: expected field") + } + // TODO detect closed + + return true, nil + } + p.query.Conditions = append(p.query.Conditions, query.Condition{Operand1: identifier, Operand1Type: query.OpField}) + p.pop() + p.step = stepWhereOperator + case stepWhereOperator: + operator := p.peek(false) + currentCondition := p.query.Conditions[len(p.query.Conditions)-1] + switch operator { + case "=": + currentCondition.Operator = query.Eq + case ">": + currentCondition.Operator = query.Gt + case ">=": + currentCondition.Operator = query.Gte + case "<": + currentCondition.Operator = query.Lt + case "<=": + currentCondition.Operator = query.Lte + case "!=": + currentCondition.Operator = query.Ne + default: + return false, newError(p.i, "at WHERE: unknown operator") + } + p.query.Conditions[len(p.query.Conditions)-1] = currentCondition + p.pop() + p.step = stepWhereValue + case stepWhereValue: + currentCondition := p.query.Conditions[len(p.query.Conditions)-1] + identifier := p.peek(false) + if isIdentifier(identifier) { + currentCondition.Operand2 = identifier + currentCondition.Operand2Type = query.OpField + } else { + quotedValue := p.peekQuotedString(false) + if p.len == 0 { + return false, newError(p.i, "at WHERE: expected quoted value") + } + currentCondition.Operand2 = quotedValue + currentCondition.Operand2Type = query.OpQuoted + } + p.query.Conditions[len(p.query.Conditions)-1] = currentCondition + p.pop() + p.step = stepWhereAnd + case stepWhereAnd: + andRWord := p.peek(true) + if andRWord != "AND" { + return false, newError(p.i, "expected AND") + } + p.pop() + p.step = stepWhereField + default: + // TODO detect closed + + return false, nil + } + } +} + func (p *parser) peekCurrent(upper bool) string { if upper { return p.sqlUpper[p.i : p.i+p.len] @@ -474,64 +533,44 @@ func (p *parser) peekIdentifierWithLength(upper bool) (string, int) { func (p *parser) validate() error { if len(p.query.Conditions) == 0 && p.step == stepWhereField { - return fmt.Errorf("at WHERE: empty WHERE clause") + return newError(p.i, "at WHERE: empty WHERE clause") } if p.query.Type == query.UnknownType { - return fmt.Errorf("query type cannot be empty") + return newError(p.i, "query type cannot be empty") } if (p.query.Type != query.Select || len(p.query.Fields) == 0) && p.query.TableName == "" { - return fmt.Errorf("table name cannot be empty") + return newError(p.i, "table name cannot be empty") } if len(p.query.Conditions) == 0 && (p.query.Type == query.Update || p.query.Type == query.Delete) { - return fmt.Errorf("at WHERE: WHERE clause is mandatory for UPDATE & DELETE") + return newError(p.i, "at WHERE: WHERE clause is mandatory for UPDATE & DELETE") } for _, c := range p.query.Conditions { if c.Operator == query.UnknownOperator { - return fmt.Errorf("at WHERE: condition without operator") + return newError(p.i, "at WHERE: condition without operator") } - if c.Operand1 == "" && c.Operand1IsField { - return fmt.Errorf("at WHERE: condition with empty left side operand") + if c.Operand1 == "" && c.Operand1Type == query.OpField { + return newError(p.i, "at WHERE: condition with empty left side operand") } - if c.Operand2 == "" && c.Operand2IsField { - return fmt.Errorf("at WHERE: condition with empty right side operand") + if c.Operand2 == "" && c.Operand2Type == query.OpField { + return newError(p.i, "at WHERE: condition with empty right side operand") } } if p.query.Type == query.Insert && len(p.query.Inserts) == 0 { - return fmt.Errorf("at INSERT INTO: need at least one row to insert") + return newError(p.i, "at INSERT INTO: need at least one row to insert") } if p.query.Type == query.Insert { for _, i := range p.query.Inserts { if len(i) != len(p.query.Fields) { - return fmt.Errorf("at INSERT INTO: value count doesn't match field count") + return newError(p.i, "at INSERT INTO: value count doesn't match field count") } } } if p.query.Type == query.Select && len(p.query.Fields) != len(p.query.Aliases) { - return fmt.Errorf("fileds and aliases count mismatch") + return newError(p.i, "fileds and aliases count mismatch") } return nil } -func (p *parser) logError() { - if p.err == nil || !p.verbose { - return - } - fmt.Println(p.sql) - fmt.Println(strings.Repeat(" ", p.i) + "^") - fmt.Println(p.err) -} - -func (p *parser) FormatError() ([3]string, error) { - if p.err == nil { - return p.errArr, nil - } - p.errArr[0] = p.sql - p.errArr[1] = strings.Repeat(" ", p.i) + "^" - p.errArr[1] = p.err.Error() - - return p.errArr, p.err -} - //var regexIdentifier = regexp.MustCompile("[a-zA-Z_][a-zA-Z_0-9]*") func isIdentifier(s string) bool { diff --git a/sql_test.go b/sql_test.go index eaad94c..b6d99cc 100644 --- a/sql_test.go +++ b/sql_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "log" "os" + "strings" "testing" "text/template" @@ -17,6 +18,7 @@ type testCase struct { SQL string Expected query.Query Err error + Ended bool } type output struct { @@ -114,7 +116,7 @@ func TestSQL(t *testing.T) { TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Eq, Operand2: "", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -127,7 +129,7 @@ func TestSQL(t *testing.T) { TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Lt, Operand2: "1", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Lt, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -140,7 +142,7 @@ func TestSQL(t *testing.T) { TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Lte, Operand2: "1", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Lte, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -153,7 +155,7 @@ func TestSQL(t *testing.T) { TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Gt, Operand2: "1", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Gt, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -166,7 +168,7 @@ func TestSQL(t *testing.T) { TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Gte, Operand2: "1", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Gte, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -179,7 +181,7 @@ func TestSQL(t *testing.T) { TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Ne, Operand2: "1", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Ne, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -192,7 +194,7 @@ func TestSQL(t *testing.T) { TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Ne, Operand2: "b", Operand2IsField: true}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Ne, Operand2: "b", Operand2Type: query.OpField}, }, }, Err: nil, @@ -228,8 +230,8 @@ func TestSQL(t *testing.T) { TableName: "b", Fields: []string{"a", "c", "d"}, Aliases: []string{"", "", ""}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Ne, Operand2: "1", Operand2IsField: false}, - {Operand1: "b", Operand1IsField: true, Operator: query.Eq, Operand2: "2", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Ne, Operand2: "1", Operand2Type: query.OpQuoted}, + {Operand1: "b", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "2", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -284,7 +286,7 @@ func TestSQL(t *testing.T) { TableName: "a", Updates: map[string]string{"b": "hello"}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Eq, Operand2: "1", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -297,7 +299,7 @@ func TestSQL(t *testing.T) { TableName: "a", Updates: map[string]string{"b": "hello\\'world"}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Eq, Operand2: "1", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -310,7 +312,7 @@ func TestSQL(t *testing.T) { TableName: "a", Updates: map[string]string{"b": "hello", "c": "bye"}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Eq, Operand2: "1", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -323,8 +325,8 @@ func TestSQL(t *testing.T) { TableName: "a", Updates: map[string]string{"b": "hello", "c": "bye"}, Conditions: []query.Condition{ - {Operand1: "a", Operand1IsField: true, Operator: query.Eq, Operand2: "1", Operand2IsField: false}, - {Operand1: "b", Operand1IsField: true, Operator: query.Eq, Operand2: "789", Operand2IsField: false}, + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "1", Operand2Type: query.OpQuoted}, + {Operand1: "b", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "789", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -360,7 +362,7 @@ func TestSQL(t *testing.T) { Type: query.Delete, TableName: "a", Conditions: []query.Condition{ - {Operand1: "b", Operand1IsField: true, Operator: query.Eq, Operand2: "1", Operand2IsField: false}, + {Operand1: "b", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "1", Operand2Type: query.OpQuoted}, }, }, Err: nil, @@ -451,7 +453,13 @@ func TestSQL(t *testing.T) { output := output{Types: query.TypeString, Operators: query.OperatorString} for _, tc := range ts { t.Run(tc.Name, func(t *testing.T) { - actual, err := ParseMany([]string{tc.SQL}, true) + actual, err := ParseMany([]string{tc.SQL}) + if err != nil { + if errPos, ok := err.(*ErrorWithPos); ok { + fmt.Fprintln(os.Stderr, "") + errPos.PrintPosError(tc.SQL, os.Stderr) + } + } if tc.Err != nil && err == nil { t.Errorf("Error should have been %v", tc.Err) } @@ -459,7 +467,7 @@ func TestSQL(t *testing.T) { t.Errorf("Error should have been nil but was %v", err) } if tc.Err != nil && err != nil { - require.Equal(t, tc.Err, err, "Unexpected error") + require.Equal(t, tc.Err.Error(), err.Error(), "Unexpected error") } if len(actual) > 0 { require.Equal(t, tc.Expected, actual[0], "Query didn't match expectation") @@ -474,10 +482,81 @@ func TestSQL(t *testing.T) { createReadme(output) } +func TestWhere(t *testing.T) { + ts := []testCase{ + { + Name: "empty query fails", + SQL: "", + Expected: query.Query{}, + Err: fmt.Errorf("at WHERE: empty WHERE clause"), + Ended: true, + }, + { + Name: "WHERE a", + SQL: "a ", + Expected: query.Query{ + Conditions: []query.Condition{ + {Operand1: "a", Operand1Type: query.OpField, Operator: query.UnknownOperator, Operand2: "", Operand2Type: query.OpUnknown}, + }, + }, + Err: nil, + Ended: true, + }, + { + Name: "WHERE a = ''", + SQL: "a = ''", + Expected: query.Query{ + Conditions: []query.Condition{ + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "", Operand2Type: query.OpQuoted}, + }, + }, + Err: nil, + Ended: true, + }, + } + + output := output{Types: query.TypeString, Operators: query.OperatorString} + for _, tc := range ts { + t.Run(tc.Name, func(t *testing.T) { + var p parser + // init parser internals + p.step = stepWhereField + p.sql = tc.SQL + p.sqlUpper = strings.ToUpper(tc.SQL) + + ended, err := p.parseWhere() + if err != nil { + if errPos, ok := err.(*ErrorWithPos); ok { + fmt.Fprintln(os.Stderr, "") + errPos.PrintPosError(tc.SQL, os.Stderr) + } + } + if tc.Err != nil && err == nil { + t.Errorf("Error should have been %v", tc.Err) + } + if tc.Err == nil && err != nil { + t.Errorf("Error should have been nil but was %v", err) + } + if tc.Ended != ended { + t.Errorf("End not detected") + } + if tc.Err != nil && err != nil { + require.Equal(t, tc.Err.Error(), err.Error(), "Unexpected error") + } + require.Equal(t, tc.Expected, p.query, "Query didn't match expectation") + if tc.Err != nil { + output.ErrorExamples = append(output.ErrorExamples, tc) + } else { + output.NoErrorExamples = append(output.NoErrorExamples, tc) + } + }) + } +} + func BenchmarkSQLSelect(b *testing.B) { sql := "SELECT a AS text FROM 'b' WHERE c = 'c' AND d = 'd'" for i := 0; i < b.N; i++ { - q, err := Parse(sql, false) + q, err := Parse(sql) if err != nil { b.Errorf("Error should have been %v: %v", err, q) } @@ -487,7 +566,7 @@ func BenchmarkSQLSelect(b *testing.B) { func BenchmarkSQLInsert(b *testing.B) { sql := "INSERT INTO 'a' (b,c, d) VALUES ('1','2' , '3' )" for i := 0; i < b.N; i++ { - q, err := Parse(sql, false) + q, err := Parse(sql) if err != nil { b.Errorf("Error should have been %v: %v", err, q) } From 5c146a21ca585fdb4d3f7465119c384e30cd3ca6 Mon Sep 17 00:00:00 2001 From: msaf1980 Date: Tue, 3 Aug 2021 00:32:03 +0500 Subject: [PATCH 14/14] sqlparser: refactor for scan reserved works with map (perfomance) --- README.md | 2 +- query/query.go | 2 +- sql.go | 250 +++++++++++++++++++++++++++++++++++-------------- sql_test.go | 77 ++++++++++++--- 4 files changed, 248 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index b8e8851..6d47013 100644 --- a/README.md +++ b/README.md @@ -529,7 +529,7 @@ at SELECT: expected field to SELECT ``` query, err := sqlparser.Parse(`SELECT a AS`) -at SELECT: expected alias (AS) for a +at AS: expected alias for a ``` ### Example: SELECT with empty WHERE fails diff --git a/query/query.go b/query/query.go index 3291422..61a476d 100644 --- a/query/query.go +++ b/query/query.go @@ -73,7 +73,7 @@ const ( OpUnknown OperandType = iota OpField OpQuoted - OpUnquoted + OpNumber ) // Condition is a single boolean condition in a WHERE clause diff --git a/sql.go b/sql.go index e8f21b4..0267896 100644 --- a/sql.go +++ b/sql.go @@ -100,6 +100,7 @@ type parser struct { i int len int peeked string + peekQuoted bool sql string sqlUpper string step step @@ -129,14 +130,24 @@ func (p *parser) doParse() (query.Query, error) { case "SELECT": p.query.Type = query.Select p.step = stepSelectField - case "INSERT INTO": + case "INSERT": + p.pop() + s = p.peek(true) + if s != "INTO" { + return p.query, newErrorf(p.i, "at INSERT: expected INTO, got %s", s) + } p.query.Type = query.Insert p.step = stepInsertTable case "UPDATE": p.query.Type = query.Update p.query.Updates = map[string]string{} p.step = stepUpdateTable - case "DELETE FROM": + case "DELETE": + p.pop() + s = p.peek(true) + if s != "FROM" { + return p.query, newErrorf(p.i, "at DELETE: expected FROM, got %s", s) + } p.query.Type = query.Delete p.step = stepDeleteFromTable default: @@ -145,7 +156,7 @@ func (p *parser) doParse() (query.Query, error) { p.pop() case stepSelectField: identifier := p.peek(false) - if !isIdentifierOrAsterisk(identifier) { + if isId, _ := isIdentifierOrAsterisk(identifier); !isId { return p.query, newError(p.i, "at SELECT: expected field to SELECT") } p.query.Fields = append(p.query.Fields, identifier) @@ -155,8 +166,8 @@ func (p *parser) doParse() (query.Query, error) { // alias p.pop() alias := p.peek(false) - if !isIdentifierOrAsterisk(alias) { - return p.query, newErrorf(p.i, "at SELECT: expected alias (AS) for %s", identifier) + if isId, _ := isIdentifierOrAsterisk(alias); !isId { + return p.query, newErrorf(p.i, "at AS: expected alias for %s", identifier) } p.query.Aliases = append(p.query.Aliases, alias) p.pop() @@ -224,7 +235,7 @@ func (p *parser) doParse() (query.Query, error) { p.step = stepUpdateField case stepUpdateField: identifier := p.peek(false) - if !isIdentifier(identifier) { + if isId, _ := isIdentifier(identifier); !isId { return p.query, newError(p.i, "at UPDATE: expected at least one field to update") } p.nextUpdateField = identifier @@ -277,7 +288,7 @@ func (p *parser) doParse() (query.Query, error) { p.step = stepInsertFields case stepInsertFields: identifier := p.peek(false) - if !isIdentifier(identifier) { + if isId, _ := isIdentifier(identifier); !isId { return p.query, newError(p.i, "at INSERT INTO: expected at least one field to insert") } p.query.Fields = append(p.query.Fields, identifier) @@ -356,34 +367,39 @@ func (p *parser) parseWhere() (bool, error) { switch p.step { case stepWhereField: identifier := p.peek(false) - if len(identifier) == 0 { - return false, newError(p.i, "at WHERE: empty WHERE clause") - } else if !isIdentifier(identifier) { - if len(p.query.Conditions) == 0 { - return true, newError(p.i, "at WHERE: expected field") + if p.peekQuoted { + p.query.Conditions = append(p.query.Conditions, query.Condition{Operand1: identifier, Operand1Type: query.OpQuoted}) + } else { + if len(identifier) == 0 { + return false, newError(p.i, "at WHERE: empty WHERE clause") + } else if isId, _ := isIdentifier(identifier); !isId { + if len(p.query.Conditions) == 0 { + return true, newError(p.i, "at WHERE: expected field") + } + // TODO detect closed + + return true, nil } - // TODO detect closed - - return true, nil + p.query.Conditions = append(p.query.Conditions, query.Condition{Operand1: identifier, Operand1Type: query.OpField}) } - p.query.Conditions = append(p.query.Conditions, query.Condition{Operand1: identifier, Operand1Type: query.OpField}) p.pop() p.step = stepWhereOperator case stepWhereOperator: - operator := p.peek(false) + operatorStr := p.peek(false) currentCondition := p.query.Conditions[len(p.query.Conditions)-1] + operator, _ := reservedWords[operatorStr] switch operator { - case "=": + case rEQ: currentCondition.Operator = query.Eq - case ">": + case rGT: currentCondition.Operator = query.Gt - case ">=": + case rGTE: currentCondition.Operator = query.Gte - case "<": + case rLT: currentCondition.Operator = query.Lt - case "<=": + case rLTE: currentCondition.Operator = query.Lte - case "!=": + case rNE: currentCondition.Operator = query.Ne default: return false, newError(p.i, "at WHERE: unknown operator") @@ -394,16 +410,19 @@ func (p *parser) parseWhere() (bool, error) { case stepWhereValue: currentCondition := p.query.Conditions[len(p.query.Conditions)-1] identifier := p.peek(false) - if isIdentifier(identifier) { + if p.peekQuoted { currentCondition.Operand2 = identifier - currentCondition.Operand2Type = query.OpField + currentCondition.Operand2Type = query.OpQuoted } else { - quotedValue := p.peekQuotedString(false) - if p.len == 0 { + if isIdentifier, isNumber := isIdentifier(identifier); isIdentifier { + currentCondition.Operand2 = identifier + currentCondition.Operand2Type = query.OpField + } else if isNumber { + currentCondition.Operand2 = identifier + currentCondition.Operand2Type = query.OpNumber + } else { return false, newError(p.i, "at WHERE: expected quoted value") } - currentCondition.Operand2 = quotedValue - currentCondition.Operand2Type = query.OpQuoted } p.query.Conditions[len(p.query.Conditions)-1] = currentCondition p.pop() @@ -451,6 +470,7 @@ func (p *parser) pop() string { p.peeked = "" p.i += p.len p.len = 0 + p.peekQuoted = false p.popWhitespace() return peeked } @@ -465,35 +485,96 @@ func (p *parser) popWhitespace() { } } -var reservedWords = []string{ - "(", ")", ">=", "<=", "!=", ",", "=", ">", "<", "AS", "SELECT", "INSERT INTO", "VALUES", "UPDATE", "DELETE FROM", - "WHERE", "FROM", "SET", -} +type rWord int + +const ( + rUnknown rWord = iota + // reserwed words + rLeftBracket // "("" + rRightBracket // ")" + rGT // ">" + rGTE // ">=" + rLTE // "<=" + rLT // "<" + rEQ // "=" + rNE // "!=" + rCOMMA // "," + rSEMI //";" + rEX // "!" + rAS // "AS" + rSELECT // "SELECT" + rINSERT // "INSERT" + rINTO //"INTO" + rVALUES // "VALUES" + rUPDATE // "UPDATE" + rDELETE // "DELETE" + rWHERE // "WHERE" + rFROM // "FROM" + rSET // "SET" + r +) + +var ( + reservedSymbols = map[byte]rWord{ + '(': rLeftBracket, + ')': rRightBracket, + '>': rGT, + '<': rLT, + '=': rEQ, + '!': rEX, + ',': rCOMMA, + ';': rSEMI, + } + + reservedWords = map[string]rWord{ + "(": rLeftBracket, + ")": rRightBracket, + ">": rGT, + ">=": rGTE, + "<": rLT, + "<=": rLTE, + "=": rEQ, + "!=": rNE, + ",": rCOMMA, + ";": rSEMI, + "AS": rAS, + "SELECT": rSELECT, + "INSERT": rINSERT, + "INTO": rINTO, + "VALUES": rVALUES, + "UPDATE": rUPDATE, + "DELETE": rDELETE, + "FROM": rFROM, + "WHERE": rWHERE, + "SET": rSET, + } +) func (p *parser) peekWithLength(upper bool) (string, int) { if p.i >= len(p.sql) { return "", 0 } - for _, rWord := range reservedWords { - token := p.sqlUpper[p.i:min(len(p.sqlUpper), p.i+len(rWord))] - if token == rWord { - if !upper { - token = p.sql[p.i:min(len(p.sql), p.i+len(rWord))] - } - return token, len(token) - } - } if p.sql[p.i] == '\'' { // Quoted string return p.peekQuotedStringWithLength(upper) } + + // for _, rWord := range reservedWords { + // token := p.sqlUpper[p.i:min(len(p.sqlUpper), p.i+len(rWord))] + // if token == rWord { + // if !upper { + // token = p.sql[p.i:min(len(p.sql), p.i+len(rWord))] + // } + + // return token, len(token) + // } + // } + return p.peekIdentifierWithLength(upper) } func (p *parser) peekQuotedStringWithLength(upper bool) (string, int) { - if len(p.sql) < p.i || p.sql[p.i] != '\'' { - return "", 0 - } + p.peekQuoted = true for i := p.i + 1; i < len(p.sql); i++ { if p.sql[i] == '\'' && p.sql[i-1] != '\\' { if upper { @@ -506,12 +587,30 @@ func (p *parser) peekQuotedStringWithLength(upper bool) (string, int) { } func (p *parser) peekIdentifierWithLength(upper bool) (string, int) { - for i := p.i; i < len(p.sql); i++ { + i := p.i + if _, ok := reservedSymbols[p.sqlUpper[i]]; ok { + if p.sql[i] == '(' || p.sql[i] == ')' { + i++ + } else { + for i = p.i + 1; i < len(p.sql); i++ { + if _, ok := reservedSymbols[p.sqlUpper[i]]; !ok { + return p.sql[p.i:i], len(p.sql[p.i:i]) + } else if p.sql[i] == '(' || p.sql[i] == ')' { + break + } + } + } + return p.sql[p.i:i], len(p.sql[p.i:i]) + } + + for ; i < len(p.sql); i++ { isIdentifierSymbol := (p.sql[i] >= 'a' && p.sql[i] <= 'z') || (p.sql[i] >= 'A' && p.sql[i] <= 'Z') || (p.sql[i] >= '0' && p.sql[i] <= '9') || p.sql[i] == '*' || - p.sql[i] == '_' + p.sql[i] == '_' || + p.sql[i] == '-' || + p.sql[i] == '.' if !isIdentifierSymbol { if p.sql[i] == '(' { // detect function @@ -571,38 +670,49 @@ func (p *parser) validate() error { return nil } -//var regexIdentifier = regexp.MustCompile("[a-zA-Z_][a-zA-Z_0-9]*") - -func isIdentifier(s string) bool { +func isIdentifier(s string) (bool, bool) { if len(s) == 0 { - return false + return false, false } u := strings.ToUpper(s) - for _, rw := range reservedWords { - if u == rw { - return false - } + + if _, ok := reservedWords[u]; ok { + return false, false } - //return regexIdentifier.MatchString(s) - if (s[0] >= 'a' && s[0] <= 'z') || + + if s[0] == '-' || (s[0] >= '0' && s[0] <= '9') { + for i := 1; i < len(s); i++ { + allowedSymbol := (s[i] >= '0' && s[i] <= '9') || s[i] == '.' + if !allowedSymbol { + return false, false + } + } + return false, true + } else if (s[0] >= 'a' && s[0] <= 'z') || (s[0] >= 'A' && s[0] <= 'Z') || s[0] == '_' { - // for i := 1; i < len(s); i++ { - // isIdentifierSymbol := (s[i] >= 'a' && s[i] <= 'z') || - // (s[i] >= 'A' && s[i] <= 'Z') || - // (s[i] >= '0' && s[i] <= '9') || - // s[i] == '_' - // if !isIdentifierSymbol { - // return false - // } - // } - return true + for i := 1; i < len(s); i++ { + isIdentifierSymbol := (s[i] >= 'a' && s[i] <= 'z') || + (s[i] >= 'A' && s[i] <= 'Z') || + (s[i] >= '0' && s[i] <= '9') || + s[i] == '_' + if !isIdentifierSymbol { + if s[i] == '(' && s[len(s)-1] == ')' { + return true, false + } + return false, false + } + } + return true, false } - return false + return false, false } -func isIdentifierOrAsterisk(s string) bool { - return s == "*" || isIdentifier(s) +func isIdentifierOrAsterisk(s string) (bool, bool) { + if s == "*" { + return true, false + } + return isIdentifier(s) } func min(a, b int) int { diff --git a/sql_test.go b/sql_test.go index b6d99cc..685c767 100644 --- a/sql_test.go +++ b/sql_test.go @@ -58,7 +58,7 @@ func TestSQL(t *testing.T) { Name: "SELECT with incomplete alias fails", SQL: "SELECT a AS", Expected: query.Query{Type: query.Select}, - Err: fmt.Errorf("at SELECT: expected alias (AS) for a"), + Err: fmt.Errorf("at AS: expected alias for a"), }, { Name: "SELECT version() as version", @@ -421,10 +421,15 @@ func TestSQL(t *testing.T) { Err: nil, }, { - Name: "INSERT * fails", - SQL: "INSERT INTO 'a' (*) VALUES ('1')", - Expected: query.Query{}, - Err: fmt.Errorf("at INSERT INTO: expected at least one field to insert"), + Name: "INSERT * fails", + SQL: "INSERT INTO 'a' (*) VALUES ('1')", + Expected: query.Query{ + Type: query.Insert, + TableName: "a", + Fields: []string{"*"}, + Inserts: [][]string{{"1"}}, + }, + Err: fmt.Errorf("at INSERT INTO: expected at least one field to insert"), }, { Name: "INSERT with multiple fields works", @@ -513,9 +518,64 @@ func TestWhere(t *testing.T) { Err: nil, Ended: true, }, + { + Name: "WHERE a = 1", + SQL: "a>=1", + Expected: query.Query{ + Conditions: []query.Condition{ + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Gte, Operand2: "1", Operand2Type: query.OpNumber}, + }, + }, + Err: nil, + Ended: true, + }, + { + Name: "WHERE a = 1.24", + SQL: "a>= 1.24", + Expected: query.Query{ + Conditions: []query.Condition{ + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Gte, Operand2: "1.24", Operand2Type: query.OpNumber}, + }, + }, + Err: nil, + Ended: true, + }, + { + Name: "WHERE a = -1.21", + SQL: "a>=-1.21", + Expected: query.Query{ + Conditions: []query.Condition{ + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Gte, Operand2: "-1.21", Operand2Type: query.OpNumber}, + }, + }, + Err: nil, + Ended: true, + }, + { + Name: "WHERE a = 1 AND b > a1", + SQL: "a = 1 AND b > a1", + Expected: query.Query{ + Conditions: []query.Condition{ + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "1", Operand2Type: query.OpNumber}, + {Operand1: "b", Operand1Type: query.OpField, Operator: query.Gt, Operand2: "a1", Operand2Type: query.OpField}, + }, + }, + Err: nil, + Ended: true, + }, + { + Name: "ERROR (a1) WHERE a = 1a", + SQL: "a = 1a", + Expected: query.Query{ + Conditions: []query.Condition{ + {Operand1: "a", Operand1Type: query.OpField, Operator: query.Eq, Operand2: "", Operand2Type: query.OpUnknown}, + }, + }, + Err: fmt.Errorf("at WHERE: expected quoted value"), + Ended: false, + }, } - output := output{Types: query.TypeString, Operators: query.OperatorString} for _, tc := range ts { t.Run(tc.Name, func(t *testing.T) { var p parser @@ -544,11 +604,6 @@ func TestWhere(t *testing.T) { require.Equal(t, tc.Err.Error(), err.Error(), "Unexpected error") } require.Equal(t, tc.Expected, p.query, "Query didn't match expectation") - if tc.Err != nil { - output.ErrorExamples = append(output.ErrorExamples, tc) - } else { - output.NoErrorExamples = append(output.NoErrorExamples, tc) - } }) } }