|
|
@ -118,14 +118,13 @@ func awkTokenizer(input *string) ([]string, int) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func Tokenize(str *string, delimiter *regexp.Regexp) []Token {
|
|
|
|
func Tokenize(str *string, delimiter *regexp.Regexp) []Token {
|
|
|
|
prefixLength := 0
|
|
|
|
|
|
|
|
if delimiter == nil {
|
|
|
|
if delimiter == nil {
|
|
|
|
// AWK-style (\S+\s*)
|
|
|
|
// AWK-style (\S+\s*)
|
|
|
|
tokens, prefixLength := awkTokenizer(str)
|
|
|
|
tokens, prefixLength := awkTokenizer(str)
|
|
|
|
return withPrefixLengths(tokens, prefixLength)
|
|
|
|
return withPrefixLengths(tokens, prefixLength)
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
tokens := delimiter.FindAllString(*str, -1)
|
|
|
|
tokens := delimiter.FindAllString(*str, -1)
|
|
|
|
return withPrefixLengths(tokens, prefixLength)
|
|
|
|
return withPrefixLengths(tokens, 0)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|