2015-02-12 11:12:19 +00:00
|
|
|
package lexer
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"strings"
|
|
|
|
"unicode/utf8"
|
|
|
|
)
|
|
|
|
|
|
|
|
type (
|
|
|
|
// lexer holds the state of the scanner.
|
|
|
|
Lexer struct {
|
2015-02-16 11:18:14 +00:00
|
|
|
input string // the string being scanned
|
|
|
|
state stateFn // the next lexing function to enter
|
|
|
|
lineNum int // Line number
|
|
|
|
pos int // current position in the input
|
|
|
|
start int // start position of this item
|
|
|
|
width int // width of last rune read from input
|
|
|
|
items chan Item // channel of scanned items
|
2015-02-12 11:12:19 +00:00
|
|
|
}
|
|
|
|
|
2015-02-16 11:18:14 +00:00
|
|
|
// Item represents a token or text string returned from the scanner.
|
2015-02-16 08:54:53 +00:00
|
|
|
Item struct {
|
2015-02-16 11:18:14 +00:00
|
|
|
Token Token // The type of this item.
|
|
|
|
Pos int // The starting position, in bytes, of this item in the input string.
|
|
|
|
Val string // The value of this item.
|
2015-02-12 11:12:19 +00:00
|
|
|
}
|
|
|
|
|
2015-02-16 11:18:14 +00:00
|
|
|
// Token identifies the type of lex items.
|
|
|
|
Token int
|
|
|
|
|
|
|
|
// stateFn represents the state of the scanner as a function that returns the next state.
|
|
|
|
stateFn func(*Lexer) stateFn
|
2015-02-12 11:12:19 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// Special
|
2015-02-16 11:18:14 +00:00
|
|
|
Error Token = iota // error occurred; value is text of error
|
|
|
|
EOF
|
2015-02-12 11:12:19 +00:00
|
|
|
|
|
|
|
// Symbols
|
2015-02-16 11:18:14 +00:00
|
|
|
BraceOpen // {
|
|
|
|
BraceClose // }
|
|
|
|
BracketOpen // [
|
|
|
|
BracketClose // [
|
|
|
|
Quote // "
|
|
|
|
Colon // :
|
|
|
|
Comma // ,
|
2015-02-12 11:12:19 +00:00
|
|
|
|
|
|
|
// Types
|
2015-02-16 11:18:14 +00:00
|
|
|
Null // null
|
|
|
|
Bool // true, false
|
|
|
|
Number // 0, 2.5
|
|
|
|
String // "foo"
|
2015-02-12 11:12:19 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// lex creates a new scanner for the input string.
|
2015-02-16 09:02:31 +00:00
|
|
|
func New(input string) *Lexer {
|
2015-02-12 11:12:19 +00:00
|
|
|
l := &Lexer{
|
|
|
|
input: input,
|
2015-02-16 08:54:53 +00:00
|
|
|
items: make(chan Item),
|
2015-02-12 11:12:19 +00:00
|
|
|
}
|
|
|
|
return l
|
|
|
|
}
|
|
|
|
|
|
|
|
// run runs the state machine for the lexer.
|
|
|
|
func (l *Lexer) Run() {
|
|
|
|
for l.state = lexInitial; l.state != nil; {
|
|
|
|
l.state = l.state(l)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-16 11:18:14 +00:00
|
|
|
func (l *Lexer) NextItem() (item Item, ok bool) {
|
|
|
|
item, ok = <-l.items
|
|
|
|
return
|
2015-02-16 08:54:53 +00:00
|
|
|
}
|
|
|
|
|
2015-02-12 11:12:19 +00:00
|
|
|
//
|
|
|
|
// Lexer stuff
|
|
|
|
//
|
|
|
|
|
2015-02-16 08:54:53 +00:00
|
|
|
func (i Item) String() string {
|
2015-02-16 11:18:14 +00:00
|
|
|
switch i.Token {
|
|
|
|
case EOF:
|
2015-02-12 11:12:19 +00:00
|
|
|
return "EOF"
|
2015-02-16 11:18:14 +00:00
|
|
|
case Error:
|
|
|
|
return "Error: " + i.Val
|
|
|
|
case BraceOpen:
|
2015-02-16 08:54:53 +00:00
|
|
|
return "{"
|
2015-02-16 11:18:14 +00:00
|
|
|
case BraceClose:
|
2015-02-16 08:54:53 +00:00
|
|
|
return "}"
|
2015-02-16 11:18:14 +00:00
|
|
|
case BracketOpen:
|
2015-02-16 08:54:53 +00:00
|
|
|
return "["
|
2015-02-16 11:18:14 +00:00
|
|
|
case BracketClose:
|
2015-02-16 08:54:53 +00:00
|
|
|
return "]"
|
2015-02-16 11:18:14 +00:00
|
|
|
case Quote:
|
2015-02-16 08:54:53 +00:00
|
|
|
return "\""
|
2015-02-16 11:18:14 +00:00
|
|
|
case Colon:
|
2015-02-16 08:54:53 +00:00
|
|
|
return ":"
|
2015-02-16 11:18:14 +00:00
|
|
|
case Comma:
|
2015-02-16 08:54:53 +00:00
|
|
|
return ","
|
2015-02-16 11:18:14 +00:00
|
|
|
case Null:
|
2015-02-16 08:54:53 +00:00
|
|
|
return "NULL"
|
2015-02-16 11:18:14 +00:00
|
|
|
case Bool:
|
|
|
|
return "Bool: " + i.Val
|
|
|
|
case Number:
|
|
|
|
return "Number: " + i.Val
|
|
|
|
case String:
|
|
|
|
return "String: " + i.Val
|
2015-02-16 08:54:53 +00:00
|
|
|
default:
|
|
|
|
panic("Unreachable")
|
2015-02-12 11:12:19 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// next returns the next rune in the input.
|
|
|
|
func (l *Lexer) next() rune {
|
|
|
|
if int(l.pos) >= len(l.input) {
|
|
|
|
l.width = 0
|
2015-02-16 11:18:14 +00:00
|
|
|
return 0
|
2015-02-12 11:12:19 +00:00
|
|
|
}
|
|
|
|
r, w := utf8.DecodeRuneInString(l.input[l.pos:])
|
2015-02-16 08:54:53 +00:00
|
|
|
l.width = w
|
2015-02-12 11:12:19 +00:00
|
|
|
l.pos += l.width
|
|
|
|
return r
|
|
|
|
}
|
|
|
|
|
|
|
|
// peek returns but does not consume the next rune in the input.
|
|
|
|
func (l *Lexer) peek() rune {
|
|
|
|
r := l.next()
|
|
|
|
l.backup()
|
|
|
|
return r
|
|
|
|
}
|
|
|
|
|
|
|
|
// backup steps back one rune. Can only be called once per call of next.
|
|
|
|
func (l *Lexer) backup() {
|
|
|
|
l.pos -= l.width
|
|
|
|
}
|
|
|
|
|
|
|
|
// emit passes an item back to the client.
|
2015-02-16 11:18:14 +00:00
|
|
|
func (l *Lexer) emit(t Token) {
|
2015-02-16 08:54:53 +00:00
|
|
|
l.items <- Item{t, l.start, l.input[l.start:l.pos]}
|
2015-02-12 11:12:19 +00:00
|
|
|
l.start = l.pos
|
2015-02-16 11:18:14 +00:00
|
|
|
if t == EOF {
|
|
|
|
close(l.items)
|
|
|
|
}
|
2015-02-12 11:12:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// ignore skips over the pending input before this point.
|
|
|
|
func (l *Lexer) ignore() {
|
|
|
|
l.start = l.pos
|
|
|
|
}
|
|
|
|
|
2015-02-16 08:54:53 +00:00
|
|
|
func (l *Lexer) acceptString(s string) (ok bool) {
|
|
|
|
if strings.HasPrefix(l.input[l.pos:], s) {
|
|
|
|
l.pos += len(s)
|
2015-02-12 11:12:19 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
|
2015-02-16 11:18:14 +00:00
|
|
|
l.items <- Item{Error, l.start, fmt.Sprintf(format, args...)}
|
|
|
|
return nil // Stop lexing
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// States
|
|
|
|
//
|
|
|
|
|
|
|
|
func lexInitial(l *Lexer) stateFn {
|
|
|
|
for {
|
|
|
|
switch r := l.next(); r {
|
|
|
|
case ' ', '\t':
|
|
|
|
return lexSpace(l)
|
|
|
|
case '\n':
|
|
|
|
l.lineNum++
|
2015-02-16 11:25:33 +00:00
|
|
|
l.ignore()
|
2015-02-16 11:18:14 +00:00
|
|
|
case 'n':
|
|
|
|
l.backup()
|
|
|
|
return lexNull(l)
|
|
|
|
case 't', 'f':
|
|
|
|
l.backup()
|
|
|
|
return lexBool(l)
|
|
|
|
case '1', '2', '3', '4', '5', '6', '7', '8', '9', '0':
|
|
|
|
l.backup()
|
|
|
|
return lexNumber(l)
|
|
|
|
case '"':
|
|
|
|
return lexString(l)
|
|
|
|
case '[':
|
|
|
|
l.emit(BracketOpen)
|
|
|
|
case ']':
|
|
|
|
l.emit(BracketClose)
|
|
|
|
case '{':
|
|
|
|
l.emit(BraceOpen)
|
|
|
|
case '}':
|
|
|
|
l.emit(BraceClose)
|
|
|
|
case ':':
|
|
|
|
l.emit(Colon)
|
|
|
|
case ',':
|
|
|
|
l.emit(Comma)
|
|
|
|
case 0:
|
2015-02-16 11:25:33 +00:00
|
|
|
l.emit(EOF)
|
|
|
|
return nil
|
2015-02-16 11:18:14 +00:00
|
|
|
default:
|
|
|
|
panic("Unexpected symbol: " + string(r))
|
|
|
|
}
|
|
|
|
}
|
2015-02-12 11:12:19 +00:00
|
|
|
}
|
2015-02-16 11:18:14 +00:00
|
|
|
|
|
|
|
// Skip all spaces
|
|
|
|
func lexSpace(l *Lexer) stateFn {
|
|
|
|
for {
|
|
|
|
if r := l.peek(); r == ' ' || r == '\t' {
|
|
|
|
l.next()
|
|
|
|
} else {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
l.ignore()
|
|
|
|
|
|
|
|
return lexInitial
|
|
|
|
}
|
|
|
|
|
|
|
|
func lexNull(l *Lexer) stateFn {
|
|
|
|
if l.acceptString("null") {
|
|
|
|
l.emit(Null)
|
|
|
|
} else {
|
|
|
|
return l.errorf("Unexpected token")
|
|
|
|
}
|
|
|
|
return lexInitial
|
|
|
|
}
|
|
|
|
|
|
|
|
func lexBool(l *Lexer) stateFn {
|
|
|
|
if l.acceptString("true") || l.acceptString("false") {
|
|
|
|
l.emit(Bool)
|
|
|
|
}
|
|
|
|
return lexInitial
|
|
|
|
}
|
|
|
|
|
|
|
|
func lexNumber(l *Lexer) stateFn {
|
|
|
|
hasDot := false
|
|
|
|
for {
|
|
|
|
switch r := l.peek(); r {
|
|
|
|
case '1', '2', '3', '4', '5', '6', '7', '8', '9', '0':
|
|
|
|
l.next()
|
|
|
|
case '.':
|
|
|
|
if hasDot {
|
|
|
|
return l.errorf("Invalid number")
|
|
|
|
} else {
|
|
|
|
hasDot = true
|
|
|
|
l.next()
|
|
|
|
}
|
|
|
|
default:
|
2015-02-16 11:25:33 +00:00
|
|
|
l.emit(Number)
|
|
|
|
return lexInitial
|
2015-02-16 11:18:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func lexString(l *Lexer) stateFn {
|
2015-02-16 17:05:49 +00:00
|
|
|
l.ignore()
|
2015-02-16 11:18:14 +00:00
|
|
|
escaped := false
|
|
|
|
for {
|
|
|
|
switch r := l.next(); r {
|
|
|
|
case '\\':
|
2015-02-16 11:25:33 +00:00
|
|
|
escaped = !escaped
|
2015-02-16 11:18:14 +00:00
|
|
|
case '"':
|
|
|
|
if escaped {
|
|
|
|
escaped = false
|
|
|
|
} else {
|
2015-02-16 17:05:49 +00:00
|
|
|
l.backup() // Going before closing quote
|
2015-02-16 11:18:14 +00:00
|
|
|
l.emit(String)
|
2015-02-16 17:05:49 +00:00
|
|
|
l.next() // Skipping closing quote
|
2015-02-16 11:25:33 +00:00
|
|
|
return lexInitial
|
2015-02-16 11:18:14 +00:00
|
|
|
}
|
|
|
|
case '\n':
|
|
|
|
l.lineNum++
|
|
|
|
case 0:
|
|
|
|
return l.errorf("Unterminated string")
|
|
|
|
default:
|
|
|
|
escaped = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|