1
0
Fork 0
kifflom/lexer/lexer.go

298 lines
5.3 KiB
Go
Raw Normal View History

2015-02-16 17:50:16 +00:00
// This lexer is based on ideas and code presented in the talk by Rob Pike
// called "Lexical Scanning in Go". More info could be found in Golang's Blog:
// http://blog.golang.org/two-go-talks-lexical-scanning-in-go-and
2015-02-12 11:12:19 +00:00
package lexer
import (
"fmt"
"strings"
"unicode/utf8"
)
type (
2015-02-16 17:50:16 +00:00
// Holds the state of the scanner
2015-02-12 11:12:19 +00:00
Lexer struct {
2015-02-16 17:50:16 +00:00
input string // The string being scanned
2015-02-16 11:18:14 +00:00
lineNum int // Line number
2015-02-16 17:50:16 +00:00
pos int // Current position in the input
start int // Start position of this item
width int // Width of last rune read from input
items chan Item // Channel of scanned items
2015-02-12 11:12:19 +00:00
}
2015-02-16 17:50:16 +00:00
// Represents a token returned from the scanner
2015-02-16 08:54:53 +00:00
Item struct {
2015-02-16 17:50:16 +00:00
Token Token // The type of this item
Val string // The value of this item
2015-02-16 19:04:17 +00:00
Pos int // The starting position, in bytes, of this item in the input string
2015-02-12 11:12:19 +00:00
}
2015-02-16 17:50:16 +00:00
// Identifies the type of the item
2015-02-16 11:18:14 +00:00
Token int
2015-02-16 17:50:16 +00:00
// Represents the state of the scanner as a function that returns the next state
2015-02-16 11:18:14 +00:00
stateFn func(*Lexer) stateFn
2015-02-12 11:12:19 +00:00
)
const (
// Special
2015-02-16 17:50:16 +00:00
Error Token = iota
2015-02-16 11:18:14 +00:00
EOF
2015-02-12 11:12:19 +00:00
// Symbols
2015-02-16 11:18:14 +00:00
BraceOpen // {
BraceClose // }
BracketOpen // [
BracketClose // [
Quote // "
Colon // :
Comma // ,
2015-02-12 11:12:19 +00:00
// Types
2015-02-16 17:50:16 +00:00
Null
Bool
Number
String
2015-02-12 11:12:19 +00:00
)
2015-02-16 17:50:16 +00:00
// Creates a new scanner for the input string
2015-02-16 09:02:31 +00:00
func New(input string) *Lexer {
2015-02-16 17:50:16 +00:00
return &Lexer{
2015-02-12 11:12:19 +00:00
input: input,
2015-02-16 08:54:53 +00:00
items: make(chan Item),
2015-02-12 11:12:19 +00:00
}
}
2015-02-16 17:50:16 +00:00
// Starts the state machine for the lexer
2015-02-12 11:12:19 +00:00
func (l *Lexer) Run() {
2015-02-16 17:23:27 +00:00
for state := lexInitial; state != nil; {
state = state(l)
2015-02-12 11:12:19 +00:00
}
}
2015-02-16 17:50:16 +00:00
// Returns the next scanned item and a boolean, which is false on EOF
2015-02-16 11:18:14 +00:00
func (l *Lexer) NextItem() (item Item, ok bool) {
item, ok = <-l.items
return
2015-02-16 08:54:53 +00:00
}
2015-02-16 17:50:16 +00:00
// Returns the next rune in the input
2015-02-12 11:12:19 +00:00
func (l *Lexer) next() rune {
if int(l.pos) >= len(l.input) {
l.width = 0
2015-02-16 11:18:14 +00:00
return 0
2015-02-12 11:12:19 +00:00
}
r, w := utf8.DecodeRuneInString(l.input[l.pos:])
2015-02-16 08:54:53 +00:00
l.width = w
2015-02-12 11:12:19 +00:00
l.pos += l.width
return r
}
2015-02-16 17:50:16 +00:00
// Returns but does not consume the next rune in the input
2015-02-12 11:12:19 +00:00
func (l *Lexer) peek() rune {
r := l.next()
l.backup()
return r
}
2015-02-16 17:50:16 +00:00
// Tells if the following input matches the given string
func (l *Lexer) acceptString(s string) (ok bool) {
if strings.HasPrefix(l.input[l.pos:], s) {
l.pos += len(s)
return true
}
return false
}
// Steps back one rune
2015-02-12 11:12:19 +00:00
func (l *Lexer) backup() {
l.pos -= l.width
}
2015-02-16 17:50:16 +00:00
// Skips over the pending input before this point
func (l *Lexer) ignore() {
l.start = l.pos
}
// Passes an item back to the client
2015-02-16 11:18:14 +00:00
func (l *Lexer) emit(t Token) {
2015-02-16 19:04:17 +00:00
l.items <- Item{
Token: t,
Val: l.input[l.start:l.pos],
Pos: l.start,
}
2015-02-12 11:12:19 +00:00
l.start = l.pos
2015-02-16 11:18:14 +00:00
if t == EOF {
close(l.items)
}
2015-02-12 11:12:19 +00:00
}
2015-02-16 17:50:16 +00:00
// Emits an error token with given string as a value and stops lexing
2015-02-12 11:12:19 +00:00
func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
2015-02-16 19:04:17 +00:00
l.items <- Item{
Token: Error,
Val: fmt.Sprintf(format, args...),
Pos: l.start,
}
close(l.items)
2015-02-16 17:50:16 +00:00
return nil
2015-02-16 11:18:14 +00:00
}
//
// States
//
func lexInitial(l *Lexer) stateFn {
for {
switch r := l.next(); r {
case ' ', '\t':
return lexSpace(l)
case '\n':
l.lineNum++
2015-02-16 11:25:33 +00:00
l.ignore()
2015-02-16 11:18:14 +00:00
case 'n':
l.backup()
return lexNull(l)
case 't', 'f':
l.backup()
return lexBool(l)
case '1', '2', '3', '4', '5', '6', '7', '8', '9', '0':
l.backup()
return lexNumber(l)
case '"':
return lexString(l)
case '[':
l.emit(BracketOpen)
case ']':
l.emit(BracketClose)
case '{':
l.emit(BraceOpen)
case '}':
l.emit(BraceClose)
case ':':
l.emit(Colon)
case ',':
l.emit(Comma)
case 0:
2015-02-16 11:25:33 +00:00
l.emit(EOF)
return nil
2015-02-16 11:18:14 +00:00
default:
2015-02-16 19:04:17 +00:00
return l.errorf("Unexpected symbol: %c", r)
2015-02-16 11:18:14 +00:00
}
}
2015-02-12 11:12:19 +00:00
}
2015-02-16 11:18:14 +00:00
2015-02-16 17:50:16 +00:00
// Skips all spaces in the input until a visible character is found
2015-02-16 11:18:14 +00:00
func lexSpace(l *Lexer) stateFn {
for {
2015-02-16 17:50:16 +00:00
if r := l.next(); r != ' ' && r != '\t' {
l.backup()
2015-02-16 11:18:14 +00:00
break
}
}
l.ignore()
return lexInitial
}
func lexNull(l *Lexer) stateFn {
if l.acceptString("null") {
l.emit(Null)
} else {
return l.errorf("Unexpected token")
}
return lexInitial
}
func lexBool(l *Lexer) stateFn {
if l.acceptString("true") || l.acceptString("false") {
l.emit(Bool)
}
return lexInitial
}
func lexNumber(l *Lexer) stateFn {
2015-02-16 19:04:17 +00:00
var (
last rune
numDots = 0
)
2015-02-16 11:18:14 +00:00
for {
2015-02-16 17:50:16 +00:00
switch r := l.next(); r {
2015-02-16 11:18:14 +00:00
case '1', '2', '3', '4', '5', '6', '7', '8', '9', '0':
2015-02-16 19:04:17 +00:00
last = r
2015-02-16 11:18:14 +00:00
case '.':
2015-02-16 17:50:16 +00:00
numDots++
2015-02-16 19:04:17 +00:00
last = r
2015-02-16 17:50:16 +00:00
default:
l.backup()
2015-02-16 19:04:17 +00:00
if numDots > 1 || last == '.' {
2015-02-16 11:18:14 +00:00
return l.errorf("Invalid number")
}
2015-02-16 11:25:33 +00:00
l.emit(Number)
return lexInitial
2015-02-16 11:18:14 +00:00
}
}
}
func lexString(l *Lexer) stateFn {
2015-02-16 19:04:17 +00:00
// Skipping opening quote
l.ignore()
2015-02-16 11:18:14 +00:00
escaped := false
for {
switch r := l.next(); r {
case '\\':
2015-02-16 11:25:33 +00:00
escaped = !escaped
2015-02-16 11:18:14 +00:00
case '"':
if escaped {
escaped = false
} else {
2015-02-16 19:04:17 +00:00
// Going before closing quote and emitting
l.backup()
2015-02-16 11:18:14 +00:00
l.emit(String)
2015-02-16 19:04:17 +00:00
// Skipping closing quote
l.next()
l.ignore()
2015-02-16 11:25:33 +00:00
return lexInitial
2015-02-16 11:18:14 +00:00
}
case '\n':
l.lineNum++
case 0:
return l.errorf("Unterminated string")
default:
escaped = false
}
}
}
2015-02-16 17:50:16 +00:00
func (i Item) String() string {
switch i.Token {
case EOF:
return "EOF"
case Error:
return "Error: " + i.Val
case BraceOpen:
return "{"
case BraceClose:
return "}"
case BracketOpen:
return "["
case BracketClose:
return "]"
case Quote:
return "\""
case Colon:
return ":"
case Comma:
return ","
case Null:
return "NULL"
case Bool:
return "Bool: " + i.Val
case Number:
return "Number: " + i.Val
case String:
return "String: " + i.Val
default:
panic("Unreachable")
}
}