From 2ee15bc355edab4c8d88e2d3da3de13a38462a3e Mon Sep 17 00:00:00 2001 From: Gregory Eremin Date: Tue, 17 Feb 2015 02:04:17 +0700 Subject: [PATCH] Lexer tests --- lexer/lexer.go | 34 +++++++--- lexer/lexer_test.go | 161 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 8 deletions(-) create mode 100644 lexer/lexer_test.go diff --git a/lexer/lexer.go b/lexer/lexer.go index 78bc361..03aa6f5 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -23,8 +23,8 @@ type ( // Represents a token returned from the scanner Item struct { Token Token // The type of this item - Pos int // The starting position, in bytes, of this item in the input string Val string // The value of this item + Pos int // The starting position, in bytes, of this item in the input string } // Identifies the type of the item @@ -116,7 +116,11 @@ func (l *Lexer) ignore() { // Passes an item back to the client func (l *Lexer) emit(t Token) { - l.items <- Item{t, l.start, l.input[l.start:l.pos]} + l.items <- Item{ + Token: t, + Val: l.input[l.start:l.pos], + Pos: l.start, + } l.start = l.pos if t == EOF { close(l.items) @@ -125,7 +129,12 @@ func (l *Lexer) emit(t Token) { // Emits an error token with given string as a value and stops lexing func (l *Lexer) errorf(format string, args ...interface{}) stateFn { - l.items <- Item{Error, l.start, fmt.Sprintf(format, args...)} + l.items <- Item{ + Token: Error, + Val: fmt.Sprintf(format, args...), + Pos: l.start, + } + close(l.items) return nil } @@ -168,7 +177,7 @@ func lexInitial(l *Lexer) stateFn { l.emit(EOF) return nil default: - panic("Unexpected symbol: " + string(r)) + return l.errorf("Unexpected symbol: %c", r) } } } @@ -202,15 +211,20 @@ func lexBool(l *Lexer) stateFn { } func lexNumber(l *Lexer) stateFn { - numDots := 0 + var ( + last rune + numDots = 0 + ) for { switch r := l.next(); r { case '1', '2', '3', '4', '5', '6', '7', '8', '9', '0': + last = r case '.': numDots++ + last = r default: l.backup() - if numDots > 1 { + if numDots > 1 || last == '.' { return l.errorf("Invalid number") } l.emit(Number) @@ -220,6 +234,7 @@ func lexNumber(l *Lexer) stateFn { } func lexString(l *Lexer) stateFn { + // Skipping opening quote l.ignore() escaped := false for { @@ -230,9 +245,12 @@ func lexString(l *Lexer) stateFn { if escaped { escaped = false } else { - l.backup() // Going before closing quote + // Going before closing quote and emitting + l.backup() l.emit(String) - l.next() // Skipping closing quote + // Skipping closing quote + l.next() + l.ignore() return lexInitial } case '\n': diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go new file mode 100644 index 0000000..acdb8cd --- /dev/null +++ b/lexer/lexer_test.go @@ -0,0 +1,161 @@ +package lexer + +import "testing" + +func TestEmpty(t *testing.T) { + compare(t, lex(""), []Item{ + Item{EOF, "", 0}, + }) +} + +func TestNull(t *testing.T) { + compare(t, lex("null"), []Item{ + Item{Null, "null", 0}, + Item{EOF, "", 0}, + }) +} + +func TesBool(t *testing.T) { + compare(t, lex("true"), []Item{ + Item{Bool, "true", 0}, + Item{EOF, "", 0}, + }) + compare(t, lex("false"), []Item{ + Item{Bool, "false", 0}, + Item{EOF, "", 0}, + }) +} + +func TestString(t *testing.T) { + compare(t, lex("\"foo\""), []Item{ + Item{String, "foo", 0}, + Item{EOF, "", 0}, + }) +} + +func TestNumber(t *testing.T) { + compare(t, lex("123"), []Item{ + Item{Number, "123", 0}, + Item{EOF, "", 0}, + }) + compare(t, lex("123.456"), []Item{ + Item{Number, "123.456", 0}, + Item{EOF, "", 0}, + }) + compare(t, lex("123.456.789"), []Item{ + Item{Error, "Invalid number", 0}, + }) + compare(t, lex("123."), []Item{ + Item{Error, "Invalid number", 0}, + }) +} + +func TestArray(t *testing.T) { + compare(t, lex("[1, \"2\", 3]"), []Item{ + Item{BracketOpen, "[", 0}, + Item{Number, "1", 0}, + Item{Comma, ",", 0}, + Item{String, "2", 0}, + Item{Comma, ",", 0}, + Item{Number, "3", 0}, + Item{BracketClose, "]", 0}, + Item{EOF, "", 0}, + }) +} + +func TestObject(t *testing.T) { + compare(t, lex("{\"a\": 1, \"b\": 2}"), []Item{ + Item{BraceOpen, "{", 0}, + Item{String, "a", 0}, + Item{Colon, ":", 0}, + Item{Number, "1", 0}, + Item{Comma, ",", 0}, + Item{String, "b", 0}, + Item{Colon, ":", 0}, + Item{Number, "2", 0}, + Item{BraceClose, "}", 0}, + Item{EOF, "", 0}, + }) +} + +// Yay! +func TestEverything(t *testing.T) { + input := ` +{ + "foo": true, + "bar": false, + "zilch": null, + "numbers": [1, 23, 4.56, 7.89], + "bullshit": { + "nothing": "anything" + }! +} +` + compare(t, lex(input), []Item{ + Item{BraceOpen, "{", 0}, + Item{String, "foo", 0}, + Item{Colon, ":", 0}, + Item{Bool, "true", 0}, + Item{Comma, ",", 0}, + Item{String, "bar", 0}, + Item{Colon, ":", 0}, + Item{Bool, "false", 0}, + Item{Comma, ",", 0}, + Item{String, "zilch", 0}, + Item{Colon, ":", 0}, + Item{Null, "null", 0}, + Item{Comma, ",", 0}, + Item{String, "numbers", 0}, + Item{Colon, ":", 0}, + Item{BracketOpen, "[", 0}, + Item{Number, "1", 0}, + Item{Comma, ",", 0}, + Item{Number, "23", 0}, + Item{Comma, ",", 0}, + Item{Number, "4.56", 0}, + Item{Comma, ",", 0}, + Item{Number, "7.89", 0}, + Item{BracketClose, "]", 0}, + Item{Comma, ",", 0}, + Item{String, "bullshit", 0}, + Item{Colon, ":", 0}, + Item{BraceOpen, "{", 0}, + Item{String, "nothing", 0}, + Item{Colon, ":", 0}, + Item{String, "anything", 0}, + Item{BraceClose, "}", 0}, + Item{Error, "Unexpected symbol: !", 0}, + }) +} + +func compare(t *testing.T, reality, expectations []Item) { + if len(reality) != len(expectations) { + t.Errorf("Expected %d tokens, got %d", len(reality), len(expectations)) + return + } + for i, exp := range expectations { + if exp.Token != reality[i].Token { + t.Errorf("Expected an %s token, got %s", exp, reality[i]) + continue + } + if exp.Val != reality[i].Val { + t.Errorf("Expected an %s token to hold value of %q, got %q", exp, exp.Val, reality[i].Val) + } + } +} + +func lex(json string) []Item { + l := New(json) + go l.Run() + + items := []Item{} + for { + if item, ok := l.NextItem(); ok { + items = append(items, item) + } else { + break + } + } + + return items +}