Initial commit

2014-10-20 22:18:58 +07:00
commit 81068376fa
18 changed files with 4496 additions and 0 deletions
@@ -0,0 +1,314 @@
+
+package lexer
+
+import(
+	"fmt"
+	"github.com/localhots/penny/token"
+)
+
+type ActionTable [NumStates] ActionRow
+
+type ActionRow struct {
+	Accept token.Type
+	Ignore string
+}
+
+func (this ActionRow) String() string {
+	return fmt.Sprintf("Accept=%d, Ignore=%s", this.Accept, this.Ignore)
+}
+
+var ActTab = ActionTable{
+ 	ActionRow{ // S0
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S1
+		Accept: -1,
+ 		Ignore: "!whitespace",
+ 	},
+ 	ActionRow{ // S2
+		Accept: 7,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S3
+		Accept: 39,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S4
+		Accept: 9,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S5
+		Accept: 10,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S6
+		Accept: 3,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S7
+		Accept: 40,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S8
+		Accept: 28,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S9
+		Accept: 4,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S10
+		Accept: 30,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S11
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S12
+		Accept: 13,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S13
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S14
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S15
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S16
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S17
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S18
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S19
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S20
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S21
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S22
+		Accept: 24,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S23
+		Accept: 8,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S24
+		Accept: 25,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S25
+		Accept: 38,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S26
+		Accept: 5,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S27
+		Accept: 16,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S28
+		Accept: 29,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S29
+		Accept: 35,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S30
+		Accept: 33,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S31
+		Accept: 31,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S32
+		Accept: 32,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S33
+		Accept: 34,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S34
+		Accept: 37,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S35
+		Accept: 13,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S36
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S37
+		Accept: 2,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S38
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S39
+		Accept: 26,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S40
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S41
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S42
+		Accept: 19,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S43
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S44
+		Accept: 17,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S45
+		Accept: 12,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S46
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S47
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S48
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S49
+		Accept: 6,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S50
+		Accept: 36,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S51
+		Accept: 13,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S52
+		Accept: 2,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S53
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S54
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S55
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S56
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S57
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S58
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S59
+		Accept: 11,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S60
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S61
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S62
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S63
+		Accept: 14,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S64
+		Accept: 27,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S65
+		Accept: 20,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S66
+		Accept: 21,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S67
+		Accept: 15,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S68
+		Accept: 18,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S69
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S70
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S71
+		Accept: 23,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S72
+		Accept: 22,
+ 		Ignore: "",
+ 	},
+ 		
+}
@@ -0,0 +1,233 @@
+
+package lexer
+
+import (
+	
+	// "fmt"
+	// "github.com/localhots/penny/util"
+	
+	"io/ioutil"
+	"unicode/utf8"
+	"github.com/localhots/penny/token"
+)
+
+const(
+	NoState = -1
+	NumStates = 73
+	NumSymbols = 90
+) 
+
+type Lexer struct {
+	src             []byte
+	pos             int
+	line            int
+	column          int
+}
+
+func NewLexer(src []byte) *Lexer {
+	lexer := &Lexer{
+		src:    src,
+		pos:    0,
+		line:   1,
+		column: 1,
+	}
+	return lexer
+}
+
+func NewLexerFile(fpath string) (*Lexer, error) {
+	src, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return nil, err
+	}
+	return NewLexer(src), nil
+}
+
+func (this *Lexer) Scan() (tok *token.Token) {
+	
+	// fmt.Printf("Lexer.Scan() pos=%d\n", this.pos)
+	
+	tok = new(token.Token)
+	if this.pos >= len(this.src) {
+		tok.Type = token.EOF
+		tok.Pos.Offset, tok.Pos.Line, tok.Pos.Column = this.pos, this.line, this.column
+		return
+	}
+	start, end := this.pos, 0
+	tok.Type = token.INVALID
+	state, rune1, size := 0, rune(-1), 0
+	for state != -1 {
+	
+		// fmt.Printf("\tpos=%d, line=%d, col=%d, state=%d\n", this.pos, this.line, this.column, state)
+	
+		if this.pos >= len(this.src) {
+			rune1 = -1
+		} else {
+			rune1, size = utf8.DecodeRune(this.src[this.pos:])
+			this.pos += size
+		}
+		switch rune1 {
+		case '\n':
+			this.line++
+			this.column = 1
+		case '\r':
+			this.column = 1
+		case '\t':
+			this.column += 4
+		default:
+			this.column++
+		}
+
+	
+		// Production start
+		if rune1 != -1 {
+			state = TransTab[state](rune1)
+		} else {
+			state = -1
+		}
+		// Production end
+
+		// Debug start
+		// nextState := -1
+		// if rune1 != -1 {
+		// 	nextState = TransTab[state](rune1)
+		// }
+		// fmt.Printf("\tS%d, : tok=%s, rune == %s(%x), next state == %d\n", state, token.TokMap.Id(tok.Type), util.RuneToString(rune1), rune1, nextState)
+		// fmt.Printf("\t\tpos=%d, size=%d, start=%d, end=%d\n", this.pos, size, start, end)
+		// if nextState != -1 {
+		// 	fmt.Printf("\t\taction:%s\n", ActTab[nextState].String())
+		// }
+		// state = nextState
+		// Debug end
+	
+
+		if state != -1 {
+			switch {
+			case ActTab[state].Accept != -1:
+				tok.Type = ActTab[state].Accept
+				// fmt.Printf("\t Accept(%s), %s(%d)\n", string(act), token.TokMap.Id(tok), tok)
+				end = this.pos
+			case ActTab[state].Ignore != "":
+				// fmt.Printf("\t Ignore(%s)\n", string(act))
+				start = this.pos
+				state = 0
+				if start >= len(this.src) {
+					tok.Type = token.EOF
+				}
+
+			}
+		} else {
+			if tok.Type == token.INVALID {
+				end = this.pos
+			}
+		}
+	}
+	if end > start {
+		this.pos = end
+		tok.Lit = this.src[start:end]
+	} else {
+		tok.Lit = []byte{}
+	}
+	tok.Pos.Offset = start
+	tok.Pos.Column = this.column
+	tok.Pos.Line = this.line
+	return
+}
+
+func (this *Lexer) Reset() {
+	this.pos = 0
+}
+
+/*
+Lexer symbols:
+0: '_'
+1: '_'
+2: '_'
+3: '='
+4: '&'
+5: '&'
+6: '|'
+7: '|'
+8: '!'
+9: '|'
+10: '('
+11: ')'
+12: 'f'
+13: 'o'
+14: 'r'
+15: 'i'
+16: 'n'
+17: 'c'
+18: 'a'
+19: 's'
+20: 'e'
+21: 'e'
+22: 's'
+23: 'a'
+24: 'c'
+25: ';'
+26: ';'
+27: 'i'
+28: 'f'
+29: 't'
+30: 'h'
+31: 'e'
+32: 'n'
+33: 'f'
+34: 'i'
+35: 'e'
+36: 'l'
+37: 'i'
+38: 'f'
+39: 'e'
+40: 'l'
+41: 's'
+42: 'e'
+43: 'w'
+44: 'h'
+45: 'i'
+46: 'l'
+47: 'e'
+48: 'u'
+49: 'n'
+50: 't'
+51: 'i'
+52: 'l'
+53: '{'
+54: '}'
+55: 'd'
+56: 'o'
+57: 'd'
+58: 'o'
+59: 'n'
+60: 'e'
+61: '<'
+62: '<'
+63: '&'
+64: '>'
+65: '>'
+66: '&'
+67: '>'
+68: '>'
+69: '<'
+70: '>'
+71: '>'
+72: '|'
+73: '<'
+74: '<'
+75: '<'
+76: '<'
+77: '-'
+78: '\'
+79: 'n'
+80: '&'
+81: ';'
+82: ' '
+83: '\t'
+84: '\n'
+85: '\r'
+86: 'a'-'z'
+87: 'A'-'Z'
+88: '0'-'9'
+89: .
+
+*/