Establish complete source spans and diagnostics

parser/ast.go

@@ -2942,7 +2942,7 @@ func (q *QueryParam) Pos() Pos {
 }
 
 func (q *QueryParam) End() Pos {
-	return q.RBracePos
+	return q.RBracePos + 1
 }
 
 func (q *QueryParam) Accept(visitor ASTVisitor) error {
@@ -3015,7 +3015,7 @@ func (f *FunctionExpr) Pos() Pos {
 }
 
 func (f *FunctionExpr) End() Pos {
-	return f.Params.RightParenPos
+	return f.Params.RightParenPos + 1
 }
 
 func (f *FunctionExpr) Accept(visitor ASTVisitor) error {

parser/error.go

@@ -70,12 +70,18 @@ func (e *ParseError) renderCaret(b *strings.Builder) {
 	line := e.starts.lineText(e.input, e.Line)
 	b.WriteString(line)
 	b.WriteByte('\n')
-	for i := 1; i < e.Column; i++ {
-		b.WriteByte(' ')
+	lineStart := e.starts[int(e.Line)-1]
+	offset := int(e.Pos) - lineStart
+	if offset < 0 {
+		offset = 0
 	}
+	if offset > len(line) {
+		offset = len(line)
+	}
+	b.WriteString(strings.Repeat(" ", displayWidth(line[:offset])))
 	width := 1
-	if e.Got != nil && len(e.Got.String) > width {
-		width = len(e.Got.String)
+	if e.Got != nil && displayWidth(e.Got.String) > width {
+		width = displayWidth(e.Got.String)
 	}
 	b.WriteString(strings.Repeat("^", width))
 	b.WriteByte('\n')

parser/error_test.go

@@ -79,6 +79,15 @@ func TestParseError_LexicalFailure(t *testing.T) {
 	}
 }
 
+func TestParseError_CaretUsesDisplayWidth(t *testing.T) {
+	_, err := NewParser("SELECT\t中文 FROM 123").ParseStmts()
+	require.Error(t, err)
+	var pe *ParseError
+	require.ErrorAs(t, err, &pe)
+	require.Equal(t, 1, pe.Line)
+	require.Contains(t, pe.Error(), "SELECT\t中文 FROM 123\n        ^")
+}
+
 func TestParser_TokenConsumptionError(t *testing.T) {
 	for _, tc := range []struct {
 		name  string

parser/lexer.go

@@ -301,8 +301,8 @@ func (l *Lexer) consumeString() error {
 	l.currentToken = &Token{
 		Kind:   TokenKindString,
 		String: l.slice(1, i),
-		Pos:    Pos(l.offset + 1),
-		End:    Pos(l.offset + i),
+		Pos:    Pos(l.offset),
+		End:    Pos(l.offset + i + 1),
 	}
 	l.skipN(i + 1)
 	return nil

parser/parser_column.go

@@ -633,10 +633,11 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen
 		return p.parseNumber(p.Pos())
 	case p.matchTokenKind(TokenKindQuestionMark):
 		// Placeholder `?`
+		curToken := p.current()
 		_ = p.lexer.consumeToken()
 		return &PlaceHolder{
-			PlaceholderPos: pos,
-			PlaceHolderEnd: pos,
+			PlaceholderPos: curToken.Pos,
+			PlaceHolderEnd: curToken.End,
 			Type:           string(TokenKindQuestionMark),
 		}, nil
 	default:
@@ -1884,12 +1885,13 @@ func (p *Parser) parseEnumValueExpr(pos Pos) (*EnumValue, error) {
 }
 
 func (p *Parser) parseColumnStar(pos Pos) (*Ident, error) {
+	curToken := p.current()
 	if err := p.expectTokenKind("*"); err != nil {
 		return nil, err
 	}
 	return &Ident{
-		NamePos: pos,
-		NameEnd: pos,
+		NamePos: curToken.Pos,
+		NameEnd: curToken.End,
 		Name:    "*",
 	}, nil
 }

parser/parser_query.go

@@ -1148,12 +1148,14 @@ func (p *Parser) parseSetOperation(selectStmt *SelectQuery) error {
 				return err
 			}
 			selectStmt.UnionAll = unionAllExpr
+			selectStmt.StatementEnd = unionAllExpr.End()
 		case p.tryConsumeKeywords(KeywordDistinct):
 			unionDistinctExpr, err := p.parseSelectQuery(p.Pos())
 			if err != nil {
 				return err
 			}
 			selectStmt.UnionDistinct = unionDistinctExpr
+			selectStmt.StatementEnd = unionDistinctExpr.End()
 		default:
 			return fmt.Errorf("expected ALL or DISTINCT, got %s", p.currentTokenKind())
 		}
@@ -1163,12 +1165,14 @@ func (p *Parser) parseSetOperation(selectStmt *SelectQuery) error {
 			return err
 		}
 		selectStmt.Except = exceptExpr
+		selectStmt.StatementEnd = exceptExpr.End()
 	case p.tryConsumeKeywords(KeywordIntersect):
 		intersectExpr, err := p.parseSelectQuery(p.Pos())
 		if err != nil {
 			return err
 		}
 		selectStmt.Intersect = intersectExpr
+		selectStmt.StatementEnd = intersectExpr.End()
 	}
 
 	return nil

parser/position_test.go

@@ -97,7 +97,8 @@ func TestKeywordArgFunctionPositions(t *testing.T) {
 	stmt := parseOneStmt(t, sql).(*SelectQuery)
 	fn := stmt.SelectItems[0].Expr.(*FunctionExpr)
 	require.Equal(t, Pos(7), fn.Pos())
-	require.Equal(t, Pos(len(sql)-1), fn.End(), "ends on the closing paren, as any call does")
+	// Spans are complete half-open source ranges, so the closing paren is included.
+	require.Equal(t, Pos(len(sql)), fn.End())
 
 	from := fn.Params.Items.Items[0].(*ColumnExpr).Expr.(*BinaryOperation)
 	require.Equal(t, "BOTH", sql[12:16])
@@ -110,8 +111,8 @@ func TestKeywordArgFunctionPositions(t *testing.T) {
 	modifier := from.LeftExpr.(*UnaryExpr)
 	require.Equal(t, Pos(12), modifier.Pos())
 	require.Equal(t, modifier.Expr.End(), modifier.End())
-	require.Equal(t, Pos(18), modifier.Expr.Pos())
-	require.Equal(t, Pos(19), modifier.Expr.End())
+	require.Equal(t, Pos(17), modifier.Expr.Pos())
+	require.Equal(t, Pos(20), modifier.Expr.End())
 }
 
 func TestJoinLocalityIsInsideTheJoinSpan(t *testing.T) {
@@ -123,3 +124,36 @@ func TestJoinLocalityIsInsideTheJoinSpan(t *testing.T) {
 	require.Equal(t, Pos(17), join.Pos())
 	require.Equal(t, Pos(len(sql)), join.End())
 }
+
+func TestSourceSpansCoverReportedNodes(t *testing.T) {
+	for _, tc := range []struct {
+		sql       string
+		wantSlice string
+		get       func(Expr) Expr
+	}{
+		{"SELECT *", "*", func(e Expr) Expr { return e.(*SelectQuery).SelectItems[0].Expr }},
+		{"SELECT ?", "?", func(e Expr) Expr { return e.(*SelectQuery).SelectItems[0].Expr }},
+		{"SELECT 'abc'", "'abc'", func(e Expr) Expr { return e.(*SelectQuery).SelectItems[0].Expr }},
+		{"SELECT f(1)", "f(1)", func(e Expr) Expr { return e.(*SelectQuery).SelectItems[0].Expr }},
+	} {
+		t.Run(tc.sql, func(t *testing.T) {
+			stmt := parseOneStmt(t, tc.sql)
+			node := tc.get(stmt)
+			require.GreaterOrEqual(t, node.Pos(), Pos(0))
+			require.LessOrEqual(t, node.Pos(), node.End())
+			require.LessOrEqual(t, node.End(), Pos(len(tc.sql)))
+			require.Equal(t, tc.wantSlice, tc.sql[node.Pos():node.End()])
+		})
+	}
+}
+
+func TestSelectQuerySpanContainsSetOperations(t *testing.T) {
+	sql := "SELECT 1 INTERSECT SELECT 2 UNION ALL SELECT 3"
+	root := parseOneStmt(t, sql).(*SelectQuery)
+	require.Equal(t, Pos(len(sql)), root.End())
+	require.NotNil(t, root.Intersect)
+	require.NotNil(t, root.Intersect.UnionAll)
+	require.LessOrEqual(t, root.Pos(), root.Intersect.Pos())
+	require.LessOrEqual(t, root.Intersect.End(), root.End())
+	require.LessOrEqual(t, root.Intersect.UnionAll.End(), root.End())
+}

parser/source.go

@@ -1,6 +1,10 @@
 package parser
 
-import "sort"
+import (
+	"sort"
+	"unicode"
+	"unicode/utf8"
+)
 
 // lineStarts holds the byte offset where each line of the input begins
 // (lineStarts[i] is the start of 0-based line i). It is built once so that
@@ -45,3 +49,29 @@ func (s lineStarts) lineText(input string, line int) string {
 	}
 	return input[start:end]
 }
+
+func displayWidth(text string) int {
+	width := 0
+	for len(text) > 0 {
+		r, size := utf8.DecodeRuneInString(text)
+		switch {
+		case r == '\t':
+			width += 8 - width%8
+		case unicode.Is(unicode.Mn, r) || unicode.Is(unicode.Me, r):
+		case eastAsianWide(r):
+			width += 2
+		default:
+			width++
+		}
+		text = text[size:]
+	}
+	return width
+}
+
+func eastAsianWide(r rune) bool {
+	return r >= 0x1100 && (r <= 0x115f || r == 0x2329 || r == 0x232a ||
+		(r >= 0x2e80 && r <= 0xa4cf) || (r >= 0xac00 && r <= 0xd7a3) ||
+		(r >= 0xf900 && r <= 0xfaff) || (r >= 0xfe10 && r <= 0xfe19) ||
+		(r >= 0xfe30 && r <= 0xfe6f) || (r >= 0xff00 && r <= 0xff60) ||
+		(r >= 0xffe0 && r <= 0xffe6))
+}

parser/testdata/basic/output/quantile_functions.sql.golden.json

@@ -1,7 +1,7 @@
 [
   {
     "SelectPos": 0,
-    "StatementEnd": 43,
+    "StatementEnd": 44,
     "With": null,
     "Top": null,
     "HasDistinct": false,

parser/testdata/basic/output/set_statement.sql.golden.json

@@ -50,7 +50,7 @@
     "SetPos": 81,
     "Settings": {
       "SettingsPos": 85,
-      "ListEnd": 132,
+      "ListEnd": 133,
       "Items": [
         {
           "SettingsPos": 85,
@@ -61,8 +61,8 @@
             "NameEnd": 124
           },
           "Expr": {
-            "LiteralPos": 128,
-            "LiteralEnd": 132,
+            "LiteralPos": 127,
+            "LiteralEnd": 133,
             "Literal": "true"
           }
         }

parser/testdata/basic/output/settings_statement.sql.golden.json

@@ -50,7 +50,7 @@
     "SetPos": 91,
     "Settings": {
       "SettingsPos": 100,
-      "ListEnd": 147,
+      "ListEnd": 148,
       "Items": [
         {
           "SettingsPos": 100,
@@ -61,8 +61,8 @@
             "NameEnd": 139
           },
           "Expr": {
-            "LiteralPos": 143,
-            "LiteralEnd": 147,
+            "LiteralPos": 142,
+            "LiteralEnd": 148,
             "Literal": "true"
           }
         }

parser/testdata/basic/output/statement_boundaries.sql.golden.json

@@ -1,7 +1,7 @@
 [
   {
     "SelectPos": 0,
-    "StatementEnd": 34,
+    "StatementEnd": 35,
     "With": null,
     "Top": null,
     "HasDistinct": false,