kifflom/lexer/lexer_test.go

package lexer

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"runtime"
	"testing"

	"github.com/localhots/kifflom/buffer"
)

func BenchmarkRun(t *testing.B) {
	f, _ := os.Open("big1.json")
	b, _ := ioutil.ReadAll(f)
	t.ReportAllocs()
	for i := 0; i < t.N; i++ {
		lex(b)
	}
}

func BenchmarkStandardJSON(t *testing.B) {
	f, _ := os.Open("big1.json")
	b, _ := ioutil.ReadAll(f)
	t.ReportAllocs()
	for i := 0; i < t.N; i++ {
		var res interface{}
		json.Unmarshal(b, &res)
	}
}

func TestEmpty(t *testing.T) {
	compare(t, lex([]byte("")), []Item{
		{EOF, "\x00", 0, 0},
	})
}

func TestNull(t *testing.T) {
	compare(t, lex([]byte("null")), []Item{
		{Null, "null", 0, 0},
		{EOF, "\x00", 0, 0},
	})
}

func TesBool(t *testing.T) {
	compare(t, lex([]byte("true")), []Item{
		{Bool, "true", 0, 0},
		{EOF, "\x00", 0, 0},
	})
	compare(t, lex([]byte("false")), []Item{
		{Bool, "false", 0, 0},
		{EOF, "\x00", 0, 0},
	})
}

func TestString(t *testing.T) {
	compare(t, lex([]byte(`"foo"`)), []Item{
		{String, "foo", 0, 0},
		{EOF, "\x00", 0, 0},
	})
}

func TestNumber(t *testing.T) {
	compare(t, lex([]byte("123")), []Item{
		{Number, "123", 0, 0},
		{EOF, "\x00", 0, 0},
	})
	compare(t, lex([]byte("123.456")), []Item{
		{Number, "123.456", 0, 0},
		{EOF, "\x00", 0, 0},
	})
	compare(t, lex([]byte("123.456.789")), []Item{
		{Error, `Invalid number: "123.456.789"`, 0, 0},
	})
	compare(t, lex([]byte("123.")), []Item{
		{Error, `Invalid number: "123."`, 0, 0},
	})
}

func TestArray(t *testing.T) {
	compare(t, lex([]byte(`[1, "2", 3]`)), []Item{
		{BracketOpen, "[", 0, 0},
		{Number, "1", 0, 0},
		{Comma, ",", 0, 0},
		{String, "2", 0, 0},
		{Comma, ",", 0, 0},
		{Number, "3", 0, 0},
		{BracketClose, "]", 0, 0},
		{EOF, "\x00", 0, 0},
	})
}

func TestObject(t *testing.T) {
	compare(t, lex([]byte(`{"a": 1, "b": 2}`)), []Item{
		{BraceOpen, "{", 0, 0},
		{String, "a", 0, 0},
		{Colon, ":", 0, 0},
		{Number, "1", 0, 0},
		{Comma, ",", 0, 0},
		{String, "b", 0, 0},
		{Colon, ":", 0, 0},
		{Number, "2", 0, 0},
		{BraceClose, "}", 0, 0},
		{EOF, "\x00", 0, 0},
	})
}

// Yay!
func TestEverything(t *testing.T) {
	input := []byte(`{
	    "foo": true,
	    "bar": false,
	    "zilch": null,
	    "numbers": [1, 23, 4.56, 7.89],
	    "bullshit": {
	        "nothing": "anything"
	    }!
	}`)

	compare(t, lex(input), []Item{
		{BraceOpen, "{", 0, 0},
		{String, "foo", 0, 0},
		{Colon, ":", 0, 0},
		{Bool, "true", 0, 0},
		{Comma, ",", 0, 0},
		{String, "bar", 0, 0},
		{Colon, ":", 0, 0},
		{Bool, "false", 0, 0},
		{Comma, ",", 0, 0},
		{String, "zilch", 0, 0},
		{Colon, ":", 0, 0},
		{Null, "null", 0, 0},
		{Comma, ",", 0, 0},
		{String, "numbers", 0, 0},
		{Colon, ":", 0, 0},
		{BracketOpen, "[", 0, 0},
		{Number, "1", 0, 0},
		{Comma, ",", 0, 0},
		{Number, "23", 0, 0},
		{Comma, ",", 0, 0},
		{Number, "4.56", 0, 0},
		{Comma, ",", 0, 0},
		{Number, "7.89", 0, 0},
		{BracketClose, "]", 0, 0},
		{Comma, ",", 0, 0},
		{String, "bullshit", 0, 0},
		{Colon, ":", 0, 0},
		{BraceOpen, "{", 0, 0},
		{String, "nothing", 0, 0},
		{Colon, ":", 0, 0},
		{String, "anything", 0, 0},
		{BraceClose, "}", 0, 0},
		{Error, "Unexpected symbol: '!'", 0, 0},
	})
}

func compare(t *testing.T, reality, expectations []Item) {
	if len(reality) != len(expectations) {
		t.Errorf("Expected %d tokens, got %d", len(reality), len(expectations))
		t.Error(runtime.Caller(1))
		return
	}
	for i, exp := range expectations {
		if exp.Token != reality[i].Token {
			t.Errorf("Expected an %s token, got %s", exp, reality[i])
			t.Error(runtime.Caller(1))
			continue
		}
		if exp.Val != reality[i].Val {
			t.Errorf("Expected an %s token to hold value of %q, got %q", exp, exp.Val, reality[i].Val)
			t.Error(runtime.Caller(1))
		}
	}
}

func lex(b []byte) []Item {
	buf := buffer.NewBytesBuffer(b)
	lex := New(buf)
	go lex.Run()

	items := []Item{}
	for {
		if item, ok := lex.NextItem(); ok {
			items = append(items, item)
		} else {
			break
		}
	}

	return items
}