ast.Errors

This commit introduces ast.Errors- an error type that should be used
whenever a generated error refers to a place in the query source. This
commit switches the parser package to use ast.Error but a follow up
commit will use ast.Errors for the semantic package as well.

api/client/connection.go

@@ -16,14 +16,16 @@ import (
 	"github.com/brimdata/zed"
 	"github.com/brimdata/zed/api"
 	"github.com/brimdata/zed/api/client/auth0"
-	"github.com/brimdata/zed/compiler/parser"
+	"github.com/brimdata/zed/compiler/ast"
 	"github.com/brimdata/zed/lake"
 	"github.com/brimdata/zed/lake/branches"
 	"github.com/brimdata/zed/lakeparse"
+	"github.com/brimdata/zed/pkg/unpack"
 	"github.com/brimdata/zed/runtime/exec"
 	"github.com/brimdata/zed/zio/zngio"
 	"github.com/brimdata/zed/zson"
 	"github.com/segmentio/ksuid"
+	"go.uber.org/multierr"
 )
 
 const (
@@ -292,27 +294,33 @@ func (c *Connection) Revert(ctx context.Context, poolID ksuid.KSUID, branchName
 	return commit, err
 }
 
+var queryErrUnpacker = unpack.New(ast.Error{})
+
 // Query assembles a query from src and filenames and runs it.
 //
 // As for Connection.Do, if the returned error is nil, the user is expected to
 // call Response.Body.Close.
-func (c *Connection) Query(ctx context.Context, head *lakeparse.Commitish, src string, filenames ...string) (*Response, error) {
-	src, srcInfo, err := parser.ConcatSource(filenames, src)
-	if err != nil {
-		return nil, err
-	}
+func (c *Connection) Query(ctx context.Context, head *lakeparse.Commitish, src string) (*Response, error) {
 	body := api.QueryRequest{Query: src}
 	if head != nil {
 		body.Head = *head
 	}
 	req := c.NewRequest(ctx, http.MethodPost, "/query?ctrl=T", body)
 	res, err := c.Do(req)
 	var ae *api.Error
-	if errors.As(err, &ae) {
-		if m, ok := ae.Info.(map[string]interface{}); ok {
-			if offset, ok := m["parse_error_offset"].(float64); ok {
-				return res, parser.NewError(src, srcInfo, int(offset))
+	if errors.As(err, &ae) && ae.Info != nil {
+		if list, ok := ae.Info.([]any); ok {
+			var errs error
+			for _, l := range list {
+				var lerr *ast.Error
+				if queryErrUnpacker.UnmarshalObject(l, &lerr) != nil {
+					// If an error is encountered here just return the parent
+					// error since this is more interesting.
+					return nil, err
+				}
+				errs = multierr.Append(errs, lerr)
 			}
+			return nil, errs
 		}
 	}
 	return res, err

cli/clierrors/format.go

@@ -0,0 +1,66 @@
+package clierrors
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/brimdata/zed/compiler/ast"
+	"github.com/brimdata/zed/compiler/parser"
+	"go.uber.org/multierr"
+)
+
+func Format(set *parser.SourceSet, err error) error {
+	if err == nil {
+		return err
+	}
+	var errs []error
+	for _, err := range multierr.Errors(err) {
+		if asterr, ok := err.(*ast.Error); ok {
+			err = formatASTError(set, asterr)
+		}
+		errs = append(errs, err)
+	}
+	return errors.Join(errs...)
+}
+
+func formatASTError(set *parser.SourceSet, err *ast.Error) error {
+	src := set.SourceOf(err.Pos)
+	start := src.Position(err.Pos)
+	end := src.Position(err.End)
+	var b strings.Builder
+	fmt.Fprintf(&b, "%s (", err)
+	if src.Filename != "" {
+		fmt.Fprintf(&b, "%s: ", src.Filename)
+	}
+	line := src.LineOfPos(set.Contents, err.Pos)
+	fmt.Fprintf(&b, "line %d, column %d):\n%s\n", start.Line, start.Column, line)
+	if end.IsValid() {
+		formatSpanError(&b, line, start, end)
+	} else {
+		formatPointError(&b, start)
+	}
+	return errors.New(b.String())
+}
+
+func formatSpanError(b *strings.Builder, line string, start, end parser.Position) {
+	col := start.Column - 1
+	b.WriteString(strings.Repeat(" ", col))
+	n := len(line) - col
+	if start.Line == end.Line {
+		n = end.Column - col
+	}
+	b.WriteString(strings.Repeat("~", n))
+}
+
+func formatPointError(b *strings.Builder, start parser.Position) {
+	col := start.Column - 1
+	for k := 0; k < col; k++ {
+		if k >= col-4 && k != col-1 {
+			b.WriteByte('=')
+		} else {
+			b.WriteByte(' ')
+		}
+	}
+	b.WriteString("^ ===")
+}

cli/queryflags/flags.go

@@ -11,14 +11,15 @@ import (
 	"strings"
 
 	"github.com/brimdata/zed/cli"
-	"github.com/brimdata/zed/compiler"
+	"github.com/brimdata/zed/cli/clierrors"
 	"github.com/brimdata/zed/compiler/ast"
 	"github.com/brimdata/zed/compiler/data"
 	"github.com/brimdata/zed/compiler/parser"
 	"github.com/brimdata/zed/compiler/semantic"
 	"github.com/brimdata/zed/pkg/storage"
 	"github.com/brimdata/zed/zbuf"
 	"github.com/brimdata/zed/zson"
+	"go.uber.org/multierr"
 )
 
 type Flags struct {
@@ -32,7 +33,7 @@ func (f *Flags) SetFlags(fs *flag.FlagSet) {
 	fs.Var(&f.Includes, "I", "source file containing Zed query text (may be used multiple times)")
 }
 
-func (f *Flags) ParseSourcesAndInputs(paths []string) ([]string, ast.Seq, bool, error) {
+func (f *Flags) ParseSourcesAndInputs(paths []string) ([]string, ast.Seq, *parser.SourceSet, bool, error) {
 	var src string
 	if len(paths) != 0 && !cli.FileExists(paths[0]) && !isURLWithKnownScheme(paths[0], "http", "https", "s3") {
 		src = paths[0]
@@ -41,25 +42,34 @@ func (f *Flags) ParseSourcesAndInputs(paths []string) ([]string, ast.Seq, bool,
 			// Consider a lone argument to be a query if it compiles
 			// and appears to start with a from or yield operator.
 			// Otherwise, consider it a file.
-			query, err := compiler.Parse(src, f.Includes...)
+			query, set, err := parse(src, f.Includes...)
 			if err == nil {
 				if s, err := semantic.Analyze(context.Background(), query, data.NewSource(storage.NewLocalEngine(), nil), nil); err == nil {
 					if semantic.HasSource(s) {
-						return nil, query, false, nil
+						return nil, query, set, false, nil
 					}
 					if semantic.StartsWithYield(s) {
-						return nil, query, true, nil
+						return nil, query, set, true, nil
 					}
 				}
 			}
-			return nil, nil, false, singleArgError(src, err)
+			return nil, nil, nil, false, singleArgError(src, set, err)
 		}
 	}
-	query, err := compiler.Parse(src, f.Includes...)
+	query, set, err := parse(src, f.Includes...)
 	if err != nil {
-		return nil, nil, false, err
+		return nil, nil, set, false, clierrors.Format(set, err)
 	}
-	return paths, query, false, nil
+	return paths, query, set, false, nil
+}
+
+func parse(src string, includes ...string) (ast.Seq, *parser.SourceSet, error) {
+	set, err := parser.ConcatSource(includes, src)
+	if err != nil {
+		return nil, nil, err
+	}
+	seq, err := parser.ParseZed(string(set.Contents))
+	return seq, set, err
 }
 
 func isURLWithKnownScheme(path string, schemes ...string) bool {
@@ -80,25 +90,30 @@ func (f *Flags) PrintStats(stats zbuf.Progress) {
 	}
 }
 
-func singleArgError(src string, err error) error {
+func singleArgError(src string, set *parser.SourceSet, err error) error {
 	var b strings.Builder
 	b.WriteString("could not invoke zq with a single argument because:")
 	if len(src) > 20 {
 		src = src[:20] + "..."
 	}
 	fmt.Fprintf(&b, "\n - a file could not be found with the name %q", src)
-	var perr *parser.Error
-	if errors.As(err, &perr) {
-		b.WriteString("\n - the argument could not be compiled as a valid Zed query due to parse error (")
-		if perr.LineNum > 0 {
-			fmt.Fprintf(&b, "line %d, ", perr.LineNum)
-		}
-		fmt.Fprintf(&b, "column %d):", perr.Column)
-		for _, l := range strings.Split(perr.ParseErrorContext(), "\n") {
-			fmt.Fprintf(&b, "\n   %s", l)
+	if hasASTErrors(err) {
+		b.WriteString("\n - the argument could not be compiled as a valid Zed query:")
+		for _, line := range strings.Split(clierrors.Format(set, err).Error(), "\n") {
+			fmt.Fprintf(&b, "\n   %s", line)
 		}
 	} else {
 		b.WriteString("\n - the argument did not parse as a valid Zed query")
 	}
 	return errors.New(b.String())
 }
+
+func hasASTErrors(errs error) bool {
+	for _, err := range multierr.Errors(errs) {
+		var aerr *ast.Error
+		if errors.As(err, &aerr) {
+			return true
+		}
+	}
+	return false
+}

cli/zq/command.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/brimdata/zed"
 	"github.com/brimdata/zed/cli"
+	"github.com/brimdata/zed/cli/clierrors"
 	"github.com/brimdata/zed/cli/inputflags"
 	"github.com/brimdata/zed/cli/outputflags"
 	"github.com/brimdata/zed/cli/queryflags"
@@ -129,7 +130,7 @@ func (c *Command) Run(args []string) error {
 		// Prevent ParseSourcesAndInputs from treating args[0] as a path.
 		args = append(args, "-")
 	}
-	paths, flowgraph, null, err := c.queryFlags.ParseSourcesAndInputs(args)
+	paths, flowgraph, set, null, err := c.queryFlags.ParseSourcesAndInputs(args)
 	if err != nil {
 		return fmt.Errorf("zq: %w", err)
 	}
@@ -156,7 +157,7 @@ func (c *Command) Run(args []string) error {
 	comp := compiler.NewFileSystemCompiler(local)
 	query, err := runtime.CompileQuery(ctx, zctx, comp, flowgraph, readers)
 	if err != nil {
-		return err
+		return clierrors.Format(set, err)
 	}
 	defer query.Pull(true)
 	err = zbuf.CopyPuller(writer, query)

cmd/zed/dev/compile/command.go

@@ -5,9 +5,9 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"os"
 	"strings"
 
+	"github.com/brimdata/zed/cli/clierrors"
 	"github.com/brimdata/zed/cmd/zed/dev"
 	"github.com/brimdata/zed/cmd/zed/root"
 	"github.com/brimdata/zed/compiler"
@@ -112,25 +112,18 @@ func (c *Command) Run(args []string) error {
 			c.pigeon = true
 		}
 	}
-	var src string
-	if len(c.includes) > 0 {
-		for _, path := range c.includes {
-			b, err := os.ReadFile(path)
-			if err != nil {
-				return err
-			}
-			src += "\n" + string(b)
-		}
+	set, err := parser.ConcatSource(c.includes, strings.Join(args, " "))
+	if err != nil {
+		return err
 	}
-	src += strings.Join(args, " ")
 	var lk *lake.Root
 	if c.semantic || c.optimize || c.parallel != 0 {
 		lakeAPI, err := c.LakeFlags.Open(ctx)
 		if err == nil {
 			lk = lakeAPI.Root()
 		}
 	}
-	return c.parse(src, lk)
+	return c.parse(set, lk)
 }
 
 func (c *Command) header(msg string) {
@@ -142,33 +135,33 @@ func (c *Command) header(msg string) {
 	}
 }
 
-func (c *Command) parse(z string, lk *lake.Root) error {
+func (c *Command) parse(set *parser.SourceSet, lk *lake.Root) error {
 	if c.pigeon {
-		s, err := parsePigeon(z)
+		s, err := parsePigeon(set)
 		if err != nil {
 			return err
 		}
 		c.header("pigeon")
 		fmt.Println(s)
 	}
 	if c.proc {
-		seq, err := compiler.Parse(z)
+		seq, err := compiler.Parse(string(set.Contents))
 		if err != nil {
 			return err
 		}
 		c.header("proc")
 		c.writeAST(seq)
 	}
 	if c.semantic {
-		runtime, err := c.compile(z, lk)
+		runtime, err := c.compile(set, lk)
 		if err != nil {
 			return err
 		}
 		c.header("semantic")
 		c.writeDAG(runtime.Entry())
 	}
 	if c.optimize {
-		runtime, err := c.compile(z, lk)
+		runtime, err := c.compile(set, lk)
 		if err != nil {
 			return err
 		}
@@ -179,7 +172,7 @@ func (c *Command) parse(z string, lk *lake.Root) error {
 		c.writeDAG(runtime.Entry())
 	}
 	if c.parallel > 0 {
-		runtime, err := c.compile(z, lk)
+		runtime, err := c.compile(set, lk)
 		if err != nil {
 			return err
 		}
@@ -213,12 +206,16 @@ func (c *Command) writeDAG(seq dag.Seq) {
 	}
 }
 
-func (c *Command) compile(z string, lk *lake.Root) (*compiler.Job, error) {
-	p, err := compiler.Parse(z)
+func (c *Command) compile(set *parser.SourceSet, lk *lake.Root) (*compiler.Job, error) {
+	p, err := compiler.Parse(string(set.Contents))
 	if err != nil {
-		return nil, err
+		return nil, clierrors.Format(set, err)
 	}
-	return compiler.NewJob(runtime.DefaultContext(), p, data.NewSource(nil, lk), nil)
+	job, err := compiler.NewJob(runtime.DefaultContext(), p, data.NewSource(nil, lk), nil)
+	if err != nil {
+		return nil, clierrors.Format(set, err)
+	}
+	return job, nil
 }
 
 func normalize(b []byte) (string, error) {
@@ -253,14 +250,14 @@ func dagFmt(seq dag.Seq, canon bool) (string, error) {
 	return normalize(dagJSON)
 }
 
-func parsePigeon(z string) (string, error) {
-	ast, err := parser.Parse("", []byte(z))
+func parsePigeon(set *parser.SourceSet) (string, error) {
+	ast, err := parser.ParseZed(string(set.Contents))
 	if err != nil {
-		return "", err
+		return "", clierrors.Format(set, err)
 	}
 	goPEGJSON, err := json.Marshal(ast)
 	if err != nil {
-		return "", errors.New("go peg parser returned bad value for: " + z)
+		return "", errors.New("go peg parser returned bad value for: " + string(set.Contents))
 	}
 	return normalize(goPEGJSON)
 }