From 3faf1adbad66d3bc91a022209d16fe96f7b4209d Mon Sep 17 00:00:00 2001 From: Gregory Eremin Date: Wed, 18 Feb 2015 22:33:57 +0700 Subject: [PATCH] Green tests --- buffer/bytes.go | 2 +- lexer/lexer.go | 11 +++- lexer/lexer_test.go | 149 +++++++++++++++++++++++--------------------- parser/parser.go | 3 +- 4 files changed, 90 insertions(+), 75 deletions(-) diff --git a/buffer/bytes.go b/buffer/bytes.go index 2d161f6..5f43595 100644 --- a/buffer/bytes.go +++ b/buffer/bytes.go @@ -22,7 +22,7 @@ func NewBytesBuffer(input []byte) *BytesBuffer { func (b *BytesBuffer) Next() rune { var buf bytes.Buffer - for b.pos < b.size-1 { + for b.pos < b.size { buf.WriteByte(b.input[b.pos]) b.pos++ if ok := utf8.FullRune(buf.Bytes()); ok { diff --git a/lexer/lexer.go b/lexer/lexer.go index d871920..858235d 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -232,20 +232,25 @@ func lexBool(l *Lexer) stateFn { } func lexNumber(l *Lexer) stateFn { - numDots := 0 + var ( + numDots = 0 + prev rune + cur rune + ) for { - switch r := l.next(); r { + switch cur = l.next(); cur { case '1', '2', '3', '4', '5', '6', '7', '8', '9', '0': case '.': numDots++ default: l.backup(1) - if numDots > 1 || r == '.' { + if numDots > 1 || prev == '.' { return l.errorf("Invalid number: %q", l.val()) } l.emit(Number) return lexInitial } + prev = cur } } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 027355c..5abe0df 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -1,80 +1,85 @@ package lexer -import "testing" +import ( + "runtime" + "testing" + + "github.com/localhots/punk/buffer" +) func TestEmpty(t *testing.T) { compare(t, lex(""), []Item{ - Item{EOF, "", 0}, + Item{EOF, "\x00", 0, 0}, }) } func TestNull(t *testing.T) { compare(t, lex("null"), []Item{ - Item{Null, "null", 0}, - Item{EOF, "", 0}, + Item{Null, "null", 0, 0}, + Item{EOF, "\x00", 0, 0}, }) } func TesBool(t *testing.T) { compare(t, lex("true"), []Item{ - Item{Bool, "true", 0}, - Item{EOF, "", 0}, + Item{Bool, "true", 0, 0}, + Item{EOF, "\x00", 0, 0}, }) compare(t, lex("false"), []Item{ - Item{Bool, "false", 0}, - Item{EOF, "", 0}, + Item{Bool, "false", 0, 0}, + Item{EOF, "\x00", 0, 0}, }) } func TestString(t *testing.T) { compare(t, lex(`"foo"`), []Item{ - Item{String, "foo", 0}, - Item{EOF, "", 0}, + Item{String, "foo", 0, 0}, + Item{EOF, "\x00", 0, 0}, }) } func TestNumber(t *testing.T) { compare(t, lex("123"), []Item{ - Item{Number, "123", 0}, - Item{EOF, "", 0}, + Item{Number, "123", 0, 0}, + Item{EOF, "\x00", 0, 0}, }) compare(t, lex("123.456"), []Item{ - Item{Number, "123.456", 0}, - Item{EOF, "", 0}, + Item{Number, "123.456", 0, 0}, + Item{EOF, "\x00", 0, 0}, }) compare(t, lex("123.456.789"), []Item{ - Item{Error, `Invalid number: "123.456.789"`, 0}, + Item{Error, `Invalid number: "123.456.789"`, 0, 0}, }) compare(t, lex("123."), []Item{ - Item{Error, `Invalid number: "123."`, 0}, + Item{Error, `Invalid number: "123."`, 0, 0}, }) } func TestArray(t *testing.T) { compare(t, lex(`[1, "2", 3]`), []Item{ - Item{BracketOpen, "[", 0}, - Item{Number, "1", 0}, - Item{Comma, ",", 0}, - Item{String, "2", 0}, - Item{Comma, ",", 0}, - Item{Number, "3", 0}, - Item{BracketClose, "]", 0}, - Item{EOF, "", 0}, + Item{BracketOpen, "[", 0, 0}, + Item{Number, "1", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{String, "2", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{Number, "3", 0, 0}, + Item{BracketClose, "]", 0, 0}, + Item{EOF, "\x00", 0, 0}, }) } func TestObject(t *testing.T) { compare(t, lex(`{"a": 1, "b": 2}`), []Item{ - Item{BraceOpen, "{", 0}, - Item{String, "a", 0}, - Item{Colon, ":", 0}, - Item{Number, "1", 0}, - Item{Comma, ",", 0}, - Item{String, "b", 0}, - Item{Colon, ":", 0}, - Item{Number, "2", 0}, - Item{BraceClose, "}", 0}, - Item{EOF, "", 0}, + Item{BraceOpen, "{", 0, 0}, + Item{String, "a", 0, 0}, + Item{Colon, ":", 0, 0}, + Item{Number, "1", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{String, "b", 0, 0}, + Item{Colon, ":", 0, 0}, + Item{Number, "2", 0, 0}, + Item{BraceClose, "}", 0, 0}, + Item{EOF, "\x00", 0, 0}, }) } @@ -92,65 +97,69 @@ func TestEverything(t *testing.T) { } ` compare(t, lex(input), []Item{ - Item{BraceOpen, "{", 0}, - Item{String, "foo", 0}, - Item{Colon, ":", 0}, - Item{Bool, "true", 0}, - Item{Comma, ",", 0}, - Item{String, "bar", 0}, - Item{Colon, ":", 0}, - Item{Bool, "false", 0}, - Item{Comma, ",", 0}, - Item{String, "zilch", 0}, - Item{Colon, ":", 0}, - Item{Null, "null", 0}, - Item{Comma, ",", 0}, - Item{String, "numbers", 0}, - Item{Colon, ":", 0}, - Item{BracketOpen, "[", 0}, - Item{Number, "1", 0}, - Item{Comma, ",", 0}, - Item{Number, "23", 0}, - Item{Comma, ",", 0}, - Item{Number, "4.56", 0}, - Item{Comma, ",", 0}, - Item{Number, "7.89", 0}, - Item{BracketClose, "]", 0}, - Item{Comma, ",", 0}, - Item{String, "bullshit", 0}, - Item{Colon, ":", 0}, - Item{BraceOpen, "{", 0}, - Item{String, "nothing", 0}, - Item{Colon, ":", 0}, - Item{String, "anything", 0}, - Item{BraceClose, "}", 0}, - Item{Error, "Unexpected symbol: !", 0}, + Item{BraceOpen, "{", 0, 0}, + Item{String, "foo", 0, 0}, + Item{Colon, ":", 0, 0}, + Item{Bool, "true", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{String, "bar", 0, 0}, + Item{Colon, ":", 0, 0}, + Item{Bool, "false", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{String, "zilch", 0, 0}, + Item{Colon, ":", 0, 0}, + Item{Null, "null", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{String, "numbers", 0, 0}, + Item{Colon, ":", 0, 0}, + Item{BracketOpen, "[", 0, 0}, + Item{Number, "1", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{Number, "23", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{Number, "4.56", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{Number, "7.89", 0, 0}, + Item{BracketClose, "]", 0, 0}, + Item{Comma, ",", 0, 0}, + Item{String, "bullshit", 0, 0}, + Item{Colon, ":", 0, 0}, + Item{BraceOpen, "{", 0, 0}, + Item{String, "nothing", 0, 0}, + Item{Colon, ":", 0, 0}, + Item{String, "anything", 0, 0}, + Item{BraceClose, "}", 0, 0}, + Item{Error, "Unexpected symbol: '!'", 0, 0}, }) } func compare(t *testing.T, reality, expectations []Item) { if len(reality) != len(expectations) { t.Errorf("Expected %d tokens, got %d", len(reality), len(expectations)) + t.Error(runtime.Caller(1)) return } for i, exp := range expectations { if exp.Token != reality[i].Token { t.Errorf("Expected an %s token, got %s", exp, reality[i]) + t.Error(runtime.Caller(1)) continue } if exp.Val != reality[i].Val { t.Errorf("Expected an %s token to hold value of %q, got %q", exp, exp.Val, reality[i].Val) + t.Error(runtime.Caller(1)) } } } func lex(json string) []Item { - l := New(json) - go l.Run() + buf := buffer.NewBytesBuffer([]byte(json)) + lex := New(buf) + go lex.Run() items := []Item{} for { - if item, ok := l.NextItem(); ok { + if item, ok := lex.NextItem(); ok { items = append(items, item) } else { break diff --git a/parser/parser.go b/parser/parser.go index b755a1c..4c3b94d 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -35,7 +35,7 @@ func New(buf buffer.Bufferer, sels []string) *Parser { } } -// Starts parsing +// Parse all and return matches func (p *Parser) Parse() map[string][]interface{} { p.ParseStream() out := map[string][]interface{}{} @@ -49,6 +49,7 @@ func (p *Parser) Parse() map[string][]interface{} { return out } +// Starts parsing func (p *Parser) ParseStream() <-chan Match { go p.lex.Run() go func() {