Overhaul Expect engine, many changes

scan/is/is.go

@@ -10,6 +10,11 @@ generate the other. BPEGN is sometimes referred to informally as "Bonzai
 Scanner Expect" language as well since it is passed directly to
 scan.Expect or scan.Check.
 
+Tokens
+
+The BPEGN token strings are contained in the "tk" package can be used as
+is. See the "tk" package for details.
+
 Parameterized Struct and Set Slice Expressions
 
 BPEGN uses Go structs and slices to represent scannable expressions with
@@ -48,14 +53,14 @@ package is
 
 // ------------------------------- sets -------------------------------
 
-// Seq groups expressions into a sequence. It ensures that all
+// X (the slice) groups expressions into a sequence. It ensures that all
 // expressions appears in that specific order. If any are not the scan
 // fails. (Equal to parenthesis in PEGN.)
-type Seq []any
+type X []any
 
 // It (the slice) is a set of positive lookahead expressions. If any are
 // seen at the current cursor position the scan will proceed without
-// consuming them (unlike is.Opt and is.In). If none are found the scan
+// consuming them (unlike is.O and is.In). If none are found the scan
 // fails. This is useful when everything from one expression is wanted
 // except for a few positive exceptions. (Equal to ampersand (&) in
 // PEGN.) Also see the tk.IS token.
@@ -76,9 +81,9 @@ type Not []any
 // of the slice.
 type In []any
 
-// Opt is a set of optional advancing expressions. If any expression is
+// O is a set of optional advancing expressions. If any expression is
 // found the scan is advanced (unlike is.It, which does not advance).
-type Opt []any
+type O []any
 
 // To is a set of advancing expressions that mark an exclusive boundary
 // at which the scan should stop. The matching expression itself will
@@ -87,7 +92,7 @@ type Opt []any
 // In order to work with escaped boundaries use a negative
 // look-ahead sequence combined with the boundary:
 //
-//     is.To{s.Seq{is.Not{`\\`,`"`}}}
+//     is.To{s.X{is.Not{`\\`,`"`}}}
 //
 type To []any
 

scan/scan.go

@@ -56,10 +56,23 @@ type R struct {
 	// Cur is the active current cursor pointing to the Buf data.
 	Cur *Cur
 
+	// Last contains the previous Cur value when Scan was called.
+	Last *Cur
+
 	// Snapped contains the latest Cur when Snap was called.
 	Snapped *Cur
+
+	// State contains scanner state bitfield using a combination of the
+	// different constants. Currently, only EOD is used but others may be
+	// added as more state-modifying single-token expressions are
+	// considered (like tk.IS and tk.NOT now).
+	State int
 }
 
+const (
+	EOD = 1 << iota // reached EOD, current rune is tk.EOD
+)
+
 // New returns a newly initialized non-linear, rune-centric, buffered
 // data scanner with support for parsing data from io.Reader, string,
 // and []byte types. Returns nil and the error if any encountered during
@@ -91,6 +104,7 @@ func (s *R) Init(i any) error {
 	r, ln := utf8.DecodeRune(s.Buf) // scan first
 	if ln == 0 {
 		r = tk.EOD
+		s.State |= EOD
 		return fmt.Errorf("scanner: failed to scan first rune")
 	}
 
@@ -124,11 +138,10 @@ func (s *R) buffer(i any) error {
 	return err
 }
 
-// Scan decodes the next rune and advances the scanner cursor by one.
-// The method of scanning isn't as optimized as other scanner (for
-// example, the scanner from the bonzai/json package), but it is
-// sufficient for most high level needs.
+// Scan decodes the next rune and advances the scanner cursor by one
+// saving the last cursor into s.Last.
 func (s *R) Scan() {
+	s.Last = s.Mark()
 
 	if s.Cur.Next == s.BufLen {
 		s.Cur.Rune = tk.EOD
@@ -181,6 +194,9 @@ func (s *R) Mark() *Cur {
 	return &cp
 }
 
+// Rewind Jumps to Last.
+func (s *R) Rewind() { s.Jump(s.Last) }
+
 // Snap sets an extra internal cursor to the current cursor. See Mark.
 func (s *R) Snap() { s.Snapped = s.Mark() }
 
@@ -229,82 +245,135 @@ func (s *R) LookSlice(beg *Cur, end *Cur) string {
 	return string(s.Buf[beg.Byte:end.Next])
 }
 
-// Expect takes a variable list of parsable types including the
-// following (in order of priority):
-//
-//     string          - "foo" simple string
-//     rune            - 'f' uint32, but "rune" in errors
-//     is.Not{any...}  - negative look-ahead set (slice)
-//     is.In{any...}   - one positive look-ahead from set (slice)
-//     is.Seq{any...}  - required positive look-ahead sequence (slice)
-//     is.Opt{any...}  - optional positive look-ahead set (slice)
-//     is.Min{n,any}   - minimum positive look-aheads
-//     is.MMx{n,m,any} - minimum and maximum positive look-aheads
-//     is.X{n,any}     - exactly n positive look-aheads
-//     is.Rng{n,m}     - inclusive range from rune n to rune m (n,m)
+// The Expect method is primarily designed for ease of use quickly by
+// allowing BPEGN expressions to be coded much faster than traditional
+// parser functions. In fact, entire applications can be generated
+// automatically from BPEGN, PEGN, ABNF, EBNF, or any other meta syntax
+// language by adding additional scan.Hooks to do work besides just
+// parsing what is scanned. Generate the BPEGN Go expressions to pass to
+// Expect and code the hook callbacks. That's it. Coding in BPEGN comes
+// naturally when putting together a parser quickly and without the
+// cognitive overhead of switching between grammars. The BPEGN
+// expressions passed to Expect are 100% Go. For full documentation of
+// BPEGN expressions see the "is" package and the source of this Expect
+// method.
 //
-// Any token string from the tk subpackage can be used as well (and will
-// be noted as special in error out as opposed to simple strings. This
-// allows for very readable functional grammar parsers to be created
-// quickly without exceptional overhead from additional function calls
-// and indirection. As some have said, "it's regex without the regex."
-func (s *R) Expect(expr ...any) (*Cur, error) {
-	var beg, end *Cur
-	beg = s.Cur
-
-	for _, m := range expr {
-
-		// please keep the most common at the top
-		switch v := m.(type) {
-
-		case rune: // ------------------------------------------------------
-			if v != tk.ANY && s.Cur.Rune != v {
-				if v == tk.EOD {
-					s.Cur.Rune = tk.EOD
-					return s.Mark(), nil
-				}
-				err := s.ErrorExpected(m)
-				s.Jump(beg)
-				return nil, err
+// Warning: While it is nothing for most developers to be concerned
+// with, the Expect method does do a fair amount of functional recursion
+// for the sake of simplicity and to support BPEGN syntax. Those wishing
+// to gain the maximum performance should consider using other scan.R
+// methods instead in such cases. Developers are encouraged to do their
+// own benchmarking and perhaps start with BPEGN until they can create
+// more optimized parsers when and if necessary. Most will discover
+// other more substantial bottlenecks. The Bonzai project places
+// priority is on speed and quality of developer delivery over run-time
+// performance. Delivery time is far more costly than the minimal gains
+// in run-time performance. "Premature optimization is the root of all
+// evil," as they say.
+func (s *R) Expect(expr any) (*Cur, error) {
+	s.Last = s.Cur
+
+	// please keep the most common expressions types at the top
+
+	switch v := expr.(type) {
+
+	case rune: // ------------------------------------------------------
+		if v != tk.ANY && s.Cur.Rune != v {
+			err := s.ErrorExpected(v)
+			return nil, err
+		}
+		s.Scan()
+		return s.Last, nil
+
+	case string: // ----------------------------------------------------
+		if v == "" {
+			return s.Mark(), nil
+		}
+		// avoid the temptation to look directly at bytes since it would
+		// allow none runes to be passed within "strings"
+		for _, v := range []rune(v) {
+			if v != s.Cur.Rune {
+				return nil, s.ErrorExpected(v)
 			}
-			end = s.Mark()
 			s.Scan()
+		}
+		return s.Last, nil
+
+	case is.X: // -----------------------------------------------------
+		var err error
+		b := s.Mark()
+		m := s.Mark()
+		for _, i := range v {
+			m, err = s.Expect(i)
+			if err != nil {
+				s.Jump(b)
+				return nil, err
+			}
+		}
+		return m, nil
 
-		case string: // ----------------------------------------------------
-			if v == "" {
-				return nil, fmt.Errorf("expect: cannot parse empty string")
+	case is.O: // -------------------------------------------------------
+		for _, i := range v {
+			m, _ := s.Expect(i)
+			if m != nil {
+				return m, nil
 			}
-			for _, r := range []rune(v) {
-				if s.Cur.Rune != r {
-					err := s.ErrorExpected(r)
-					s.Jump(beg)
-					return nil, err
+		}
+
+	case is.Toi: // -----------------------------------------------------
+		back := s.Mark()
+		for s.Cur.Rune != tk.EOD {
+			for _, i := range v {
+				m, _ := s.Expect(i)
+				if m != nil {
+					return m, nil
 				}
-				end = s.Mark()
-				s.Scan()
 			}
+			s.Scan()
+		}
+		s.Jump(back)
+		return nil, s.ErrorExpected(v)
 
-		case is.Toi: // -----------------------------------------------------
-			var m *Cur
-			for m == nil && s.Cur.Rune != tk.EOD {
+	case is.To: // -----------------------------------------------------
+		m := s.Mark()
+		b4 := s.Mark()
+		for s.Cur.Rune != tk.EOD {
+			for _, i := range v {
+				b := s.Mark()
+				c, _ := s.Expect(i)
+				if c != nil {
+					s.Jump(b)
+					return b4, nil
+				}
+			}
+			b4 = s.Mark()
+			s.Scan()
+		}
+		s.Jump(m)
+		return nil, s.ErrorExpected(v)
+		/*
+			back := s.Mark()
+			b4 := s.Mark()
+			for s.Cur.Rune != tk.EOD {
 				for _, i := range v {
-					m, _ = s.check(i)
+					m, _ := s.Expect(i)
+					if m != nil {
+						return b4, nil
+					}
 				}
+				b4 = s.Mark()
 				s.Scan()
 			}
-			if m == nil {
-				err := s.ErrorExpected(v)
-				s.Jump(beg)
-				return nil, err
-			}
-			end = m
+			s.Jump(back)
+			return nil, s.ErrorExpected(v)
+		*/
 
-		case is.To: // -----------------------------------------------------
+	/*
 			var m, b4 *Cur
 		OUT:
 			for s.Cur.Rune != tk.EOD {
 				for _, i := range v {
-					m, _ = s.check(i)
+					m, _ = s.Expect(i)
 					if m != nil {
 						break OUT
 					}
@@ -316,167 +385,149 @@ func (s *R) Expect(expr ...any) (*Cur, error) {
 				err := s.ErrorExpected(v)
 				return nil, err
 			}
-			end = b4
-
-		case is.It: // ----------------------------------------------------
-			var m *Cur
-			for _, i := range v {
-				m, _ = s.check(i)
-				if m != nil {
-					break
-				}
-			}
-			if m == nil {
-				return nil, s.ErrorExpected(v)
-			}
-			end = s.Mark()
-
-		case is.Not: // ----------------------------------------------------
-			m := s.Mark()
-			for _, i := range v {
-				if c, _ := s.check(i); c != nil {
-					err := s.ErrorExpected(v, i)
-					return nil, err
-				}
-			}
-			end = m
-
-		case is.In: // -----------------------------------------------------
-			var m *Cur
-			for _, i := range v {
-				var err error
-				last := s.Mark()
-				m, err = s.Expect(i)
-				if err == nil {
-					break
-				}
-				s.Jump(last)
-			}
-			if m == nil {
-				return nil, s.ErrorExpected(v)
+			return b4, nil
+	*/
+
+	case is.It: // ----------------------------------------------------
+		var m *Cur
+		b := s.Mark()
+		for _, i := range v {
+			m, _ = s.Expect(i)
+			if m != nil {
+				break
 			}
-			end = m
+		}
+		if m == nil {
+			return nil, s.ErrorExpected(v)
+		}
+		s.Jump(b)
+		return b, nil
 
-		case is.Seq: // ----------------------------------------------------
-			m := s.Mark()
-			c, err := s.Expect(v...)
-			if err != nil {
+	case is.Not: // ----------------------------------------------------
+		m := s.Mark()
+		for _, i := range v {
+			if c, _ := s.Expect(i); c != nil {
 				s.Jump(m)
+				err := s.ErrorExpected(v, i)
 				return nil, err
 			}
-			end = c
-
-		case is.Opt: // ----------------------------------------------------
-			var m *Cur
-			for _, i := range v {
-				var err error
-				m, err = s.Expect(is.MMx{0, 1, i})
-				if err != nil {
-					s.Jump(beg)
-					return nil, s.ErrorExpected(v)
-				}
-			}
-			end = m
+		}
+		return m, nil
 
-		case is.MMx: // ----------------------------------------------------
-			c := 0
-			last := s.Mark()
+	case is.In: // -----------------------------------------------------
+		var m *Cur
+		for _, i := range v {
 			var err error
-			var m *Cur
-			for {
-				m, err = s.Expect(v.This)
-				if err != nil {
-					break
-				}
-				last = m
-				c++
-			}
-			if c == 0 && v.Min == 0 {
-				if end == nil {
-					end = last
-				}
-				continue
-			}
-			if !(v.Min <= c && c <= v.Max) {
-				s.Jump(last)
-				return nil, s.ErrorExpected(v)
+			last := s.Mark()
+			m, err = s.Expect(i)
+			if err == nil {
+				break
 			}
-			end = last
-
-		case is.Mn1: // ----------------------------------------------------
-			m, err := s.Expect(is.Min{1, v.This})
+			s.Jump(last)
+		}
+		if m == nil {
+			return nil, s.ErrorExpected(v)
+		}
+		return m, nil
+
+	case is.MMx: // ----------------------------------------------------
+		c := 0
+		last := s.Mark()
+		var err error
+		var m, end *Cur
+		for {
+			m, err = s.Expect(v.This)
 			if err != nil {
-				s.Jump(beg)
-				return nil, s.ErrorExpected(v)
-			}
-			end = m
-
-		case is.Min: // ----------------------------------------------------
-			c := 0
-			last := s.Mark()
-			var err error
-			var m *Cur
-			for {
-				m, err = s.Expect(v.This)
-				if err != nil {
-					break
-				}
-				last = m
-				c++
+				break
 			}
-			if c < v.Min {
-				s.Jump(beg)
-				return nil, s.ErrorExpected(v)
+			last = m
+			c++
+		}
+		if c == 0 && v.Min == 0 {
+			if end == nil {
+				end = last
 			}
-			end = last
+		}
+		if !(v.Min <= c && c <= v.Max) {
+			s.Jump(last)
+			return nil, s.ErrorExpected(v)
+		}
+		return end, nil
 
-		case is.N: // ------------------------------------------------------
-			m, err := s.Expect(is.MMx{v.N, v.N, v.This})
+	case is.Mn1: // ----------------------------------------------------
+		m := s.Mark()
+		c, err := s.Expect(is.Min{1, v.This})
+		if err != nil {
+			s.Jump(m)
+			return nil, s.ErrorExpected(v)
+		}
+		return c, nil
+
+	case is.Min: // ----------------------------------------------------
+		c := 0
+		last := s.Mark()
+		var err error
+		var m *Cur
+		for {
+			m, err = s.Expect(v.This)
 			if err != nil {
-				s.Jump(beg)
-				return nil, s.ErrorExpected(v)
+				break
 			}
-			end = m
+			last = m
+			c++
+		}
+		if c < v.Min {
+			s.Jump(last)
+			return nil, s.ErrorExpected(v)
+		}
+		return last, nil
 
-		case is.Any: // ----------------------------------------------------
-			for n := 0; n < v.N; n++ {
-				s.Scan()
-			}
-			end = s.Mark()
-			s.Scan()
+	case is.N: // ------------------------------------------------------
+		b := s.Mark()
+		m, err := s.Expect(is.MMx{v.N, v.N, v.This})
+		if err != nil {
+			s.Jump(b)
+			return nil, s.ErrorExpected(v)
+		}
+		return m, nil
 
-		case is.Rng: // ----------------------------------------------------
-			if !(v.First <= s.Cur.Rune && s.Cur.Rune <= v.Last) {
-				err := s.ErrorExpected(v)
-				s.Jump(beg)
-				return nil, err
-			}
-			end = s.Mark()
+	case is.Any: // ----------------------------------------------------
+		for n := 0; n < v.N; n++ {
 			s.Scan()
+		}
+		m := s.Mark()
+		s.Scan()
+		return m, nil
 
-		case Hook: // ------------------------------------------------------
-			if !v(s) {
-				return nil, fmt.Errorf(
-					"expect: hook function failed (%v)",
-					util.FuncName(v),
-				)
-			}
-			end = s.Mark()
-
-		case func(r *R) bool:
-			if !v(s) {
-				return nil, fmt.Errorf(
-					"expect: hook function failed (%v)",
-					util.FuncName(v),
-				)
-			}
-			end = s.Mark()
-
-		default: // --------------------------------------------------------
-			return nil, fmt.Errorf("expect: unexpr expression (%T)", m)
+	case is.Rng: // ----------------------------------------------------
+		if !(v.First <= s.Cur.Rune && s.Cur.Rune <= v.Last) {
+			err := s.ErrorExpected(v)
+			return nil, err
+		}
+		m := s.Mark()
+		s.Scan()
+		return m, nil
+
+	case Hook: // ------------------------------------------------------
+		if !v(s) {
+			return nil, fmt.Errorf(
+				"expect: hook function failed (%v)",
+				util.FuncName(v),
+			)
+		}
+		return s.Cur, nil
+
+	case func(r *R) bool:
+		if !v(s) {
+			return nil, fmt.Errorf(
+				"expect: hook function failed (%v)",
+				util.FuncName(v),
+			)
 		}
+		return s.Cur, nil
+
 	}
-	return end, nil
+	return nil, fmt.Errorf("unknown expression (%T)", expr)
 }
 
 // ErrorExpected returns a verbose, one-line error describing what was
@@ -497,16 +548,22 @@ func (s *R) ErrorExpected(this any, args ...any) error {
 	case rune: // otherwise will use uint32
 		msg = fmt.Sprintf(`expected rune %q`, v)
 	case is.It:
-		msg = fmt.Sprintf(`expected %q`, v)
+		if len(v) > 1 {
+			msg = fmt.Sprintf(`expected one of %q`, v)
+		} else {
+			msg = fmt.Sprintf(`expected %q`, v[0])
+		}
 	case is.Not:
 		msg = fmt.Sprintf(`unexpected %q`, args[0])
 	case is.In:
 		str := `expected one of %q`
 		msg = fmt.Sprintf(str, v)
-	case is.Seq:
-		str := `expected %q in sequence %q`
-		msg = fmt.Sprintf(str, args[0], v)
-	case is.Opt:
+	case is.X:
+		//str := `expected %q in sequence %q at %v beginning`
+		//msg = fmt.Sprintf(str, args[0], v, args[1])
+		str := `expected %q in sequence`
+		msg = fmt.Sprintf(str, v)
+	case is.O:
 		str := `expected an optional %v`
 		msg = fmt.Sprintf(str, v)
 	case is.Mn1:
@@ -524,9 +581,14 @@ func (s *R) ErrorExpected(this any, args ...any) error {
 	case is.Rng:
 		str := `expected range [%v-%v]`
 		msg = fmt.Sprintf(str, string(v.First), string(v.Last))
-	case is.To, is.Toi:
+	case is.Toi:
 		str := `%q not found`
+		if len(v) > 1 {
+			str = `none of %q found`
+		}
 		msg = fmt.Sprintf(str, v)
+	case is.To:
+		msg = fmt.Sprintf(`none of %q found`, v)
 	default:
 		msg = fmt.Sprintf(`expected %T %q`, v, v)
 	}
@@ -535,10 +597,3 @@ func (s *R) ErrorExpected(this any, args ...any) error {
 
 // NewLine delegates to interval Curs.NewLine.
 func (s *R) NewLine() { s.Cur.NewLine() }
-
-// check behaves exactly like Expect but jumps back to the original
-// cursor position after scanning for expected expression values.
-func (s *R) check(expr ...any) (*Cur, error) {
-	defer s.Jump(s.Mark())
-	return s.Expect(expr...)
-}

scan/scan_test.go

@@ -174,37 +174,94 @@ func ExampleErrorExpected() {
 	// expected []uint8 "foo" at U+0073 's' 1,1-1 (1-1)
 }
 
-func ExampleExpect_basic() {
+func ExampleExpect_rune_Success() {
 	s, _ := scan.New("some thing")
-	c, _ := s.Expect("some", ' ', "thin")
-	c.Print() // pointing to 'n' since last match
-	s.Print() // pointing to next after 'n' ('g')
-	s.Scan()  // advance one
-	s.Print() // <EOD> since no more data
+	c, _ := s.Expect('s')
+	c.Print() // just one so end and beg same
+	s.Print() // point to next scan 'o'
+	// Output:
+	// U+0073 's' 1,1-1 (1-1)
+	// U+006F 'o' 1,2-2 (2-2)
+}
+
+func ExampleExpect_rune_Fail() {
+	s, _ := scan.New("some thing")
+	_, err := s.Expect('S')
+	fmt.Println(err)
+	s.Print() // not advanced
+	// Output:
+	// expected rune 'S' at U+0073 's' 1,1-1 (1-1)
+	// U+0073 's' 1,1-1 (1-1)
+}
+
+func ExampleExpect_string() {
+	s, _ := scan.New("some thing")
+	c, _ := s.Expect("som")
+	c.Print() // same as s.ScanN(2), last is 'm'
+	s.Print() // point to next scan 'e'
+	// Output:
+	// U+006D 'm' 1,3-3 (3-3)
+	// U+0065 'e' 1,4-4 (4-4)
+}
+
+func ExampleExpect_compound_Expr_String() {
+	s, _ := scan.New("some thing")
+	c, _ := s.Expect(is.X{"some ", "thin"})
+	c.Print()
+	s.Print()
 	// Output:
 	// U+006E 'n' 1,9-9 (9-9)
 	// U+0067 'g' 1,10-10 (10-10)
-	// <EOD>
 }
 
-func ExampleExpect_lk() {
+func ExampleExpect_compound_Expr_Rune() {
 	s, _ := scan.New("some thing")
-	_, e := s.Expect(is.It{"foo"})
-	fmt.Println(e)
-	c, _ := s.Expect(is.It{"foo", 's'})
+	c, _ := s.Expect(is.X{"some", ' ', "thin"})
 	c.Print()
-	s.ScanN(2)
 	s.Print()
-	c, _ = s.Expect(is.It{is.Rng{'l', 'p'}})
-	s.Print() // not advanced
-	c, _ = s.Expect(is.Rng{'l', 'p'})
-	s.Print() // advanced
 	// Output:
-	// expected ["foo"] at U+0073 's' 1,1-1 (1-1)
+	// U+006E 'n' 1,9-9 (9-9)
+	// U+0067 'g' 1,10-10 (10-10)
+}
+
+func ExampleExpect_it_Success() {
+	s, _ := scan.New("some thing")
+	c, _ := s.Expect(is.It{"some"})
+	c.Print() // even though true, not moved
+	s.Print() // scanner also not moved
+	// Output:
+	// U+0073 's' 1,1-1 (1-1)
+	// U+0073 's' 1,1-1 (1-1)
+}
+
+func ExampleExpect_it_Success_Middle() {
+	s, _ := scan.New("some thing")
+	c, _ := s.Expect(is.X{"some", is.It{' '}})
+	c.Print() // advanced up to (but not including) ' '
+	s.Print() // scanner also not moved
+	// Output:
+	// U+0020 ' ' 1,5-5 (5-5)
+	// U+0020 ' ' 1,5-5 (5-5)
+}
+
+func ExampleExpect_it_Fail() {
+	s, _ := scan.New("some thing")
+	_, err := s.Expect(is.X{"some", is.It{"thing"}})
+	fmt.Println(err)
+	s.Print() // but scanner did get "some" so advanced
+	// Output:
+	// expected "thing" at U+0020 ' ' 1,5-5 (5-5)
+	// U+0073 's' 1,1-1 (1-1)
+}
+
+func ExampleExpect_it_Fail_X() {
+	s, _ := scan.New("some thing")
+	_, err := s.Expect(is.X{"some", is.It{"thing"}})
+	fmt.Println(err)
+	s.Print() // but scanner did get "some" so advanced
+	// Output:
+	// expected "thing" at U+0020 ' ' 1,5-5 (5-5)
 	// U+0073 's' 1,1-1 (1-1)
-	// U+006D 'm' 1,3-3 (3-3)
-	// U+006D 'm' 1,3-3 (3-3)
-	// U+0065 'e' 1,4-4 (4-4)
 }
 
 func ExampleExpect_not_Success() {
@@ -227,19 +284,19 @@ func ExampleExpect_not_Fail() {
 	// U+0073 's' 1,1-1 (1-1)
 }
 
-func ExampleExpect_not_Seq_Fail() {
+func ExampleExpect_not_X_Fail() {
 	s, _ := scan.New("some thing wonderful")
-	_, err := s.Expect(is.Seq{is.Not{`s`}, `o`})
+	_, err := s.Expect(is.X{is.Not{'s'}, 'o'})
 	fmt.Println(err) // sees the s so fails
 	s.Print()        // not advanced
 	// Output:
-	// unexpected "s" at U+0073 's' 1,1-1 (1-1)
+	// unexpected 's' at U+0073 's' 1,1-1 (1-1)
 	// U+0073 's' 1,1-1 (1-1)
 }
 
-func ExampleExpect_not_Seq_Success() {
+func ExampleExpect_not_X_Success() {
 	s, _ := scan.New("some thing wonderful")
-	c, _ := s.Expect(is.Seq{is.Not{`n`}, is.In{`som`}})
+	c, _ := s.Expect(is.X{is.Not{`n`}, is.In{`som`}})
 	c.Print() // pointing to last in match 'm'
 	s.Print() // advanced to next after match 'e'
 	// Output:
@@ -247,18 +304,15 @@ func ExampleExpect_not_Seq_Success() {
 	// U+0065 'e' 1,4-4 (4-4)
 }
 
-/*
-func ExampleExpect_not_To_Seq_Success() {
+func ExampleExpect_to_Success_Mid() {
 	s, _ := scan.New("some thing wonderful")
-	//c, _ := s.Expect(is.To{is.Seq{is.Not{`s`}, `o`}})
 	c, _ := s.Expect(is.To{"wo"})
-	c.Print() // pointing to only 'o' not preceeded by s
-	s.Print() // advanced to next after second 'o' ('n')
+	c.Print() // "wo" not inc, same as "some thing ", so ' '
+	s.Print() // advances to 'w'
 	// Output:
-	// U+006F 'o' 1,13-13 (13-13)
-	//
+	// U+0020 ' ' 1,11-11 (11-11)
+	// U+0077 'w' 1,12-12 (12-12)
 }
-*/
 
 func ExampleExpect_in() {
 	s, _ := scan.New("some thing")
@@ -281,8 +335,8 @@ func ExampleExpect_avoid_Not_with_In() {
 	c.Print() // unexpected success
 	s.Print() // advanced to 'o'
 	s.Back()
-	// use is.Seq instead
-	_, err := s.Expect(is.Seq{is.Not{'s'}, is.Rng{'a', 'z'}})
+	// use is.X instead
+	_, err := s.Expect(is.X{is.Not{'s'}, is.Rng{'a', 'z'}})
 	fmt.Println(err)
 	s.Print() // not advanced
 	// Output:
@@ -294,7 +348,7 @@ func ExampleExpect_avoid_Not_with_In() {
 
 func ExampleExpect_seq_Success() {
 	s, _ := scan.New("some thing")
-	c, _ := s.Expect(is.Seq{"some", ' ', "thin"})
+	c, _ := s.Expect(is.X{"some", ' ', "thin"})
 	c.Print() // same as "some thin", points at 'n'
 	s.Print() // advanced to 'g'
 	// Output:
@@ -304,7 +358,7 @@ func ExampleExpect_seq_Success() {
 
 func ExampleExpect_seq_Fail() {
 	s, _ := scan.New("some thing")
-	_, err := s.Expect(is.Seq{"some", "thin"})
+	_, err := s.Expect(is.X{"some", "thin"})
 	fmt.Println(err)
 	s.Print() // not advanced at all
 	// Output:
@@ -314,7 +368,7 @@ func ExampleExpect_seq_Fail() {
 
 func ExampleExpect_seq_Not_Success() {
 	s, _ := scan.New("some thing")
-	c, _ := s.Expect(is.Seq{"some", ` `, is.Not{`T`}, "thin"})
+	c, _ := s.Expect(is.X{"some", ` `, is.Not{`T`}, "thin"})
 	c.Print() // same as "some thin"
 	s.Print() // advanced to next after ('g')
 	// Output:
@@ -324,7 +378,7 @@ func ExampleExpect_seq_Not_Success() {
 
 func ExampleExpect_seq_Not_Fail() {
 	s, _ := scan.New("some Thing")
-	_, err := s.Expect(is.Seq{"some", ` `, is.Not{`T`}, "ignored"})
+	_, err := s.Expect(is.X{"some", ' ', is.Not{`T`}, "ignored"})
 	fmt.Println(err)
 	s.Print() // not advanced at all
 	// Output:
@@ -342,16 +396,6 @@ func ExampleExpect_token_ANY() {
 	// U+006F 'o' 1,2-2 (2-2)
 }
 
-func ExampleExpect_token_EOD() {
-	s, _ := scan.New("some thing")
-	c, _ := s.Expect("some", tk.EOD, "ignored")
-	c.Print() // same as "some"
-	s.Print() // advances
-	// Output:
-	// <EOD>
-	// <EOD>
-}
-
 func ExampleExpect_any_Success() {
 	s, _ := scan.New("some thing")
 	c, _ := s.Expect(is.Any{5})
@@ -362,17 +406,15 @@ func ExampleExpect_any_Success() {
 	// U+0068 'h' 1,7-7 (7-7)
 }
 
-func ExampleExpect_opt() {
+func ExampleExpect_o_Optional_Success() {
 	s, _ := scan.New("some thing")
-	c, _ := s.Expect(is.Opt{"thing", "some"})
-	c.Print()
-	s.Print()
-	c, _ = s.Expect(is.Opt{"foo"})
-	s.Print() // no change
+	//c, _ := s.Expect(is.O{"thing", "some"})
+	c, _ := s.Expect("some")
+	c.Print() // same as "some", points to 'e'
+	s.Print() // advances to space ' '
 	// Output:
 	// U+0065 'e' 1,4-4 (4-4)
 	// U+0020 ' ' 1,5-5 (5-5)
-	// U+0020 ' ' 1,5-5 (5-5)
 }
 
 func ExampleExpect_mn1() {
@@ -394,21 +436,13 @@ func ExampleExpect_mn1() {
 }
 
 func ExampleExpect_min() {
-	s, _ := scan.New("sommme thing")
-	start := s.Mark()
-	s.ScanN(2)
-	c, _ := s.Expect(is.Min{2, 'm'}) // goggles up all three
-	c.Print()
-	s.Print()
-	s.Jump(start)
-	_, e := s.Expect(is.Min{2, 's'}) // nope, only one found
-	fmt.Println(e)
-	s.Print()
+	s, _ := scan.New("sssoommme thing")
+	c, _ := s.Expect(is.Min{2, 's'})
+	c.Print() // needs 2, but will consume all three to last 's'
+	s.Print() // advances to next after ('o')
 	// Output:
-	// U+006D 'm' 1,5-5 (5-5)
-	// U+0065 'e' 1,6-6 (6-6)
-	// expected min 2 of 's' at U+006F 'o' 1,2-2 (2-2)
-	// U+006F 'o' 1,2-2 (2-2)
+	// U+0073 's' 1,3-3 (3-3)
+	// U+006F 'o' 1,4-4 (4-4)
 }
 
 func ExampleExpect_mMx() {
@@ -497,17 +531,7 @@ func ExampleExpect_hook() {
 
 func ExampleExpect_to_Success() {
 	s, _ := scan.New("some thing")
-	c, _ := s.Expect(is.To{`e`})
-	c.Print() // same as "som", points to 'm'
-	s.Print() // scanned next after ('e')
-	// Output:
-	// U+006D 'm' 1,3-3 (3-3)
-	// U+0065 'e' 1,4-4 (4-4)
-}
-
-func ExampleExpect_to_Seq_Success() {
-	s, _ := scan.New("some thing")
-	c, _ := s.Expect(is.To{is.Seq{`e`}})
+	c, _ := s.Expect(is.To{'e'})
 	c.Print() // same as "som", points to 'm'
 	s.Print() // scanned next after ('e')
 	// Output:
@@ -517,7 +541,7 @@ func ExampleExpect_to_Seq_Success() {
 
 func ExampleExpect_toi() {
 	s, _ := scan.New("some thing")
-	c, _ := s.Expect(is.Toi{`e`})
+	c, _ := s.Expect(is.Toi{'e'})
 	c.Print() // same as "some", points to 'e'
 	s.Print() // scanned next after (' ')
 	// Output:

scan/tk/tk.go

@@ -9,7 +9,8 @@ const (
 	// reached enabling functional parser functions to look for it reliably
 	// no matter what is being parsed. Since rune is alias for int32 and
 	// Unicode (currently) ends at \U+FFFD we are safe to use the largest
-	// possible valid rune value.
+	// possible valid rune value. Passing EOD directly to Expect always
+	// stops the scan where it is.
 	EOD rune = 1<<31 - (iota + 1) // max int32
 
 	// ANY represents any possible single rune similar to question mark (?)

scan/tk/tk_test.go

@@ -0,0 +1,15 @@
+package tk_test
+
+import (
+	"fmt"
+
+	"github.com/rwxrob/bonzai/scan/tk"
+)
+
+func Example() {
+	fmt.Printf("%U\n", tk.EOD)
+	fmt.Printf("%U\n", tk.ANY)
+	// Output:
+	// U+7FFFFFFF
+	// U+7FFFFFFE
+}