Lexer tests

lexer/lexer.go

@@ -23,8 +23,8 @@ type (
 	// Represents a token returned from the scanner
 	Item struct {
 		Token Token  // The type of this item
-		Pos   int    // The starting position, in bytes, of this item in the input string
 		Val   string // The value of this item
+		Pos   int    // The starting position, in bytes, of this item in the input string
 	}
 
 	// Identifies the type of the item
@@ -116,7 +116,11 @@ func (l *Lexer) ignore() {
 
 // Passes an item back to the client
 func (l *Lexer) emit(t Token) {
-	l.items <- Item{t, l.start, l.input[l.start:l.pos]}
+	l.items <- Item{
+		Token: t,
+		Val:   l.input[l.start:l.pos],
+		Pos:   l.start,
+	}
 	l.start = l.pos
 	if t == EOF {
 		close(l.items)
@@ -125,7 +129,12 @@ func (l *Lexer) emit(t Token) {
 
 // Emits an error token with given string as a value and stops lexing
 func (l *Lexer) errorf(format string, args ...interface{}) stateFn {
-	l.items <- Item{Error, l.start, fmt.Sprintf(format, args...)}
+	l.items <- Item{
+		Token: Error,
+		Val:   fmt.Sprintf(format, args...),
+		Pos:   l.start,
+	}
+	close(l.items)
 	return nil
 }
 
@@ -168,7 +177,7 @@ func lexInitial(l *Lexer) stateFn {
 			l.emit(EOF)
 			return nil
 		default:
-			panic("Unexpected symbol: " + string(r))
+			return l.errorf("Unexpected symbol: %c", r)
 		}
 	}
 }
@@ -202,15 +211,20 @@ func lexBool(l *Lexer) stateFn {
 }
 
 func lexNumber(l *Lexer) stateFn {
-	numDots := 0
+	var (
+		last    rune
+		numDots = 0
+	)
 	for {
 		switch r := l.next(); r {
 		case '1', '2', '3', '4', '5', '6', '7', '8', '9', '0':
+			last = r
 		case '.':
 			numDots++
+			last = r
 		default:
 			l.backup()
-			if numDots > 1 {
+			if numDots > 1 || last == '.' {
 				return l.errorf("Invalid number")
 			}
 			l.emit(Number)
@@ -220,6 +234,7 @@ func lexNumber(l *Lexer) stateFn {
 }
 
 func lexString(l *Lexer) stateFn {
+	// Skipping opening quote
 	l.ignore()
 	escaped := false
 	for {
@@ -230,9 +245,12 @@ func lexString(l *Lexer) stateFn {
 			if escaped {
 				escaped = false
 			} else {
-				l.backup() // Going before closing quote
+				// Going before closing quote and emitting
+				l.backup()
 				l.emit(String)
-				l.next() // Skipping closing quote
+				// Skipping closing quote
+				l.next()
+				l.ignore()
 				return lexInitial
 			}
 		case '\n':

lexer/lexer_test.go

@@ -0,0 +1,161 @@
+package lexer
+
+import "testing"
+
+func TestEmpty(t *testing.T) {
+	compare(t, lex(""), []Item{
+		Item{EOF, "", 0},
+	})
+}
+
+func TestNull(t *testing.T) {
+	compare(t, lex("null"), []Item{
+		Item{Null, "null", 0},
+		Item{EOF, "", 0},
+	})
+}
+
+func TesBool(t *testing.T) {
+	compare(t, lex("true"), []Item{
+		Item{Bool, "true", 0},
+		Item{EOF, "", 0},
+	})
+	compare(t, lex("false"), []Item{
+		Item{Bool, "false", 0},
+		Item{EOF, "", 0},
+	})
+}
+
+func TestString(t *testing.T) {
+	compare(t, lex("\"foo\""), []Item{
+		Item{String, "foo", 0},
+		Item{EOF, "", 0},
+	})
+}
+
+func TestNumber(t *testing.T) {
+	compare(t, lex("123"), []Item{
+		Item{Number, "123", 0},
+		Item{EOF, "", 0},
+	})
+	compare(t, lex("123.456"), []Item{
+		Item{Number, "123.456", 0},
+		Item{EOF, "", 0},
+	})
+	compare(t, lex("123.456.789"), []Item{
+		Item{Error, "Invalid number", 0},
+	})
+	compare(t, lex("123."), []Item{
+		Item{Error, "Invalid number", 0},
+	})
+}
+
+func TestArray(t *testing.T) {
+	compare(t, lex("[1, \"2\", 3]"), []Item{
+		Item{BracketOpen, "[", 0},
+		Item{Number, "1", 0},
+		Item{Comma, ",", 0},
+		Item{String, "2", 0},
+		Item{Comma, ",", 0},
+		Item{Number, "3", 0},
+		Item{BracketClose, "]", 0},
+		Item{EOF, "", 0},
+	})
+}
+
+func TestObject(t *testing.T) {
+	compare(t, lex("{\"a\": 1, \"b\": 2}"), []Item{
+		Item{BraceOpen, "{", 0},
+		Item{String, "a", 0},
+		Item{Colon, ":", 0},
+		Item{Number, "1", 0},
+		Item{Comma, ",", 0},
+		Item{String, "b", 0},
+		Item{Colon, ":", 0},
+		Item{Number, "2", 0},
+		Item{BraceClose, "}", 0},
+		Item{EOF, "", 0},
+	})
+}
+
+// Yay!
+func TestEverything(t *testing.T) {
+	input := `
+{
+    "foo": true,
+    "bar": false,
+    "zilch": null,
+    "numbers": [1, 23, 4.56, 7.89],
+    "bullshit": {
+        "nothing": "anything"
+    }!
+}
+`
+	compare(t, lex(input), []Item{
+		Item{BraceOpen, "{", 0},
+		Item{String, "foo", 0},
+		Item{Colon, ":", 0},
+		Item{Bool, "true", 0},
+		Item{Comma, ",", 0},
+		Item{String, "bar", 0},
+		Item{Colon, ":", 0},
+		Item{Bool, "false", 0},
+		Item{Comma, ",", 0},
+		Item{String, "zilch", 0},
+		Item{Colon, ":", 0},
+		Item{Null, "null", 0},
+		Item{Comma, ",", 0},
+		Item{String, "numbers", 0},
+		Item{Colon, ":", 0},
+		Item{BracketOpen, "[", 0},
+		Item{Number, "1", 0},
+		Item{Comma, ",", 0},
+		Item{Number, "23", 0},
+		Item{Comma, ",", 0},
+		Item{Number, "4.56", 0},
+		Item{Comma, ",", 0},
+		Item{Number, "7.89", 0},
+		Item{BracketClose, "]", 0},
+		Item{Comma, ",", 0},
+		Item{String, "bullshit", 0},
+		Item{Colon, ":", 0},
+		Item{BraceOpen, "{", 0},
+		Item{String, "nothing", 0},
+		Item{Colon, ":", 0},
+		Item{String, "anything", 0},
+		Item{BraceClose, "}", 0},
+		Item{Error, "Unexpected symbol: !", 0},
+	})
+}
+
+func compare(t *testing.T, reality, expectations []Item) {
+	if len(reality) != len(expectations) {
+		t.Errorf("Expected %d tokens, got %d", len(reality), len(expectations))
+		return
+	}
+	for i, exp := range expectations {
+		if exp.Token != reality[i].Token {
+			t.Errorf("Expected an %s token, got %s", exp, reality[i])
+			continue
+		}
+		if exp.Val != reality[i].Val {
+			t.Errorf("Expected an %s token to hold value of %q, got %q", exp, exp.Val, reality[i].Val)
+		}
+	}
+}
+
+func lex(json string) []Item {
+	l := New(json)
+	go l.Run()
+
+	items := []Item{}
+	for {
+		if item, ok := l.NextItem(); ok {
+			items = append(items, item)
+		} else {
+			break
+		}
+	}
+
+	return items
+}