Initial commit

ast/ast.go

@@ -0,0 +1,612 @@
+package ast
+
+import (
+	"strconv"
+
+	"github.com/localhots/penny/token"
+)
+
+// Basics
+
+type (
+	IoNumber int
+	Word     string
+	Attrib   interface{}
+)
+
+func NewWord(word Attrib) (Word, error) {
+	lit := word.(*token.Token).Lit
+	return Word(lit), nil
+}
+
+func NewIoNumber(number Attrib) (IoNumber, error) {
+	lit := number.(*token.Token).Lit
+	n, err := strconv.Atoi(string(lit))
+	return IoNumber(n), err
+}
+
+// Assignment
+
+type (
+	Assignment struct {
+		a, b Word
+	}
+)
+
+func NewAssignment(a, b Attrib) (*Assignment, error) {
+	return &Assignment{a.(Word), b.(Word)}, nil
+}
+
+// List
+
+type (
+	List []*AndOr
+)
+
+func NewList(ao Attrib) (List, error) {
+	return List{ao.(*AndOr)}, nil
+}
+
+func AppendToList(list Attrib, el Attrib) (List, error) {
+	l := list.(List)
+	l = append(l, el.(*AndOr))
+	return l, nil
+}
+
+// AndOr
+
+type (
+	Logic     int
+	AndOrStmt []*AndOr
+	AndOr     struct {
+		op Logic
+		p  *Pipeline
+	}
+)
+
+const (
+	L_FIRST Logic = iota
+	L_AND
+	L_OR
+)
+
+func NewAndOr(p Attrib) (AndOrStmt, error) {
+	return AndOrStmt{&AndOr{L_FIRST, p.(*Pipeline)}}, nil
+}
+
+func AppendAnd(stmt Attrib, p Attrib) (AndOrStmt, error) {
+	pp := stmt.(AndOrStmt)
+	pp = append(pp, &AndOr{L_AND, p.(*Pipeline)})
+	return pp, nil
+}
+
+func AppendOr(stmt Attrib, p Attrib) (AndOrStmt, error) {
+	pp := stmt.(AndOrStmt)
+	pp = append(pp, &AndOr{L_OR, p.(*Pipeline)})
+	return pp, nil
+}
+
+// Pipeline
+
+type (
+	Pipeline struct {
+		seq     PipeSequence
+		inverse bool
+	}
+)
+
+func NewPipeline(seq Attrib, inverse bool) (*Pipeline, error) {
+	return &Pipeline{seq.(PipeSequence), inverse}, nil
+}
+
+// PipeSequence
+
+type (
+	PipeSequence []Command
+)
+
+func NewPipeSequence(cmd Attrib) (PipeSequence, error) {
+	return PipeSequence{cmd.(*Command)}, nil
+}
+
+func AppendToPipeSequence(ps Attrib, cmd Attrib) (PipeSequence, error) {
+	ps1 := ps.(PipeSequence)
+	ps1 = append(ps1, cmd.(Command))
+	return ps1, nil
+}
+
+// Command
+
+type (
+	Command interface{}
+)
+
+func NewCommand(cmd Attrib, rl Attrib) (Command, error) {
+	return Command(cmd), nil
+}
+
+// CompoundCommand
+
+type (
+	CompoundCommand interface{}
+)
+
+func NewCompoundCommand(cmd Attrib) (CompoundCommand, error) {
+	return CompoundCommand(cmd), nil
+}
+
+// Subshell
+
+type (
+	Subshell struct {
+		cc CompoundCommand
+	}
+)
+
+func NewSubshell(cc Attrib) (*Subshell, error) {
+	return &Subshell{cc.(CompoundCommand)}, nil
+}
+
+// CompoundList
+
+type (
+	CompoundList struct {
+		term *Term
+		sep  *Separator
+	}
+)
+
+func NewCompoundList(term Attrib, sep Attrib) (*CompoundList, error) {
+	return &CompoundList{term.(*Term), sep.(*Separator)}, nil
+}
+
+// Term
+
+type (
+	Term     []*TermItem
+	TermItem struct {
+		ao  AndOrStmt
+		sep *Separator
+	}
+)
+
+func NewTerm(ao Attrib) (Term, error) {
+	return Term{&TermItem{ao.(AndOrStmt), nil}}, nil
+}
+
+func AppendToTerm(term Attrib, ao Attrib, sep Attrib) (Term, error) {
+	t := term.(Term)
+	t = append(t, &TermItem{ao.(AndOrStmt), sep.(*Separator)})
+	return t, nil
+}
+
+// ForClause
+
+type (
+	ForClause struct {
+		name Name
+		wl   Wordlist
+		dg   *DoGroup
+	}
+)
+
+func NewForClause(name Attrib, wl Attrib, dg Attrib) (*ForClause, error) {
+	return &ForClause{
+		name: name.(Name),
+		wl:   wl.(Wordlist),
+		dg:   dg.(*DoGroup),
+	}, nil
+}
+
+// Name
+
+type (
+	Name Word
+)
+
+func NewName(w Attrib) (Name, error) {
+	return Name(w.(Word)), nil
+}
+
+// Wordlist
+
+type (
+	Wordlist []Word
+)
+
+func NewWordlist(w Attrib) (Wordlist, error) {
+	return Wordlist{w.(Word)}, nil
+}
+
+func AppendToWordlist(wl Attrib, w Attrib) (Wordlist, error) {
+	wl1 := wl.(Wordlist)
+	wl1 = append(wl1, w.(Word))
+	return wl1, nil
+}
+
+// CaseClause
+
+type (
+	CaseClause struct {
+		word Word
+		cl   CaseList
+	}
+)
+
+func NewCaseClause(word Attrib, cl Attrib) (*CaseClause, error) {
+	return &CaseClause{
+		word: word.(Word),
+		cl:   cl.(CaseList),
+	}, nil
+}
+
+// CaseListNs
+// CaseList
+
+type (
+	CaseList []*CaseItem
+)
+
+func NewCaseList(ci Attrib) (CaseList, error) {
+	return CaseList{ci.(*CaseItem)}, nil
+}
+
+func AppendToCaseList(cl Attrib, ci Attrib) (CaseList, error) {
+	cl1 := cl.(CaseList)
+	cl1 = append(cl1, ci.(*CaseItem))
+	return cl1, nil
+}
+
+// CaseItemNs
+// CaseItem
+
+type (
+	CaseItem struct {
+		p  Pattern
+		cl *CompoundList
+	}
+)
+
+func NewCaseItem(p Attrib, cl Attrib) (*CaseItem, error) {
+	return &CaseItem{
+		p:  p.(Pattern),
+		cl: cl.(*CompoundList),
+	}, nil
+}
+
+// Pattern
+
+type (
+	Pattern []Word
+)
+
+func NewPattern(w Attrib) (Pattern, error) {
+	return Pattern{w.(Word)}, nil
+}
+
+func AppendToPattern(p Attrib, w Attrib) (Pattern, error) {
+	p1 := p.(Pattern)
+	p1 = append(p1, w.(Word))
+	return p1, nil
+}
+
+// IfClause
+// ElsePart
+
+type (
+	IfClause struct {
+		cond   *CompoundList
+		action *CompoundList
+		elsep  *IfClause
+	}
+)
+
+func NewIfClause(cond Attrib, action Attrib, elsep Attrib) (*IfClause, error) {
+	return &IfClause{
+		cond:   cond.(*CompoundList),
+		action: action.(*CompoundList),
+		elsep:  elsep.(*IfClause),
+	}, nil
+}
+
+// WhileClause
+
+type (
+	WhileClause struct {
+		cond *CompoundList
+		dg   *DoGroup
+	}
+)
+
+func NewWhileClause(cond Attrib, dg Attrib) (*WhileClause, error) {
+	return &WhileClause{
+		cond: cond.(*CompoundList),
+		dg:   dg.(*DoGroup),
+	}, nil
+}
+
+// UntilClause
+
+type (
+	UntilClause struct {
+		cond *CompoundList
+		dg   *DoGroup
+	}
+)
+
+func NewUntilClause(cond Attrib, dg Attrib) (*UntilClause, error) {
+	return &UntilClause{
+		cond: cond.(*CompoundList),
+		dg:   dg.(*DoGroup),
+	}, nil
+}
+
+// FunctionDefinition
+
+type (
+	FunctionDefinition struct {
+		name FunctionName
+		body *FunctionBody
+	}
+)
+
+func NewFunctionDefinition(name Attrib, body Attrib) (*FunctionDefinition, error) {
+	return &FunctionDefinition{
+		name: name.(FunctionName),
+		body: body.(*FunctionBody),
+	}, nil
+}
+
+// FunctionBody
+
+type (
+	FunctionBody struct {
+		cc CompoundCommand
+		rl RedirectList
+	}
+)
+
+func NewFunctionBody(cc Attrib, rl Attrib) (*FunctionBody, error) {
+	return &FunctionBody{
+		cc: cc.(CompoundCommand),
+		rl: rl.(RedirectList),
+	}, nil
+}
+
+// FunctionName
+
+type (
+	FunctionName Word
+)
+
+func NewFunctionName(w Attrib) (FunctionName, error) {
+	return FunctionName(w.(Word)), nil
+}
+
+// BraceGroup
+
+type (
+	BraceGroup struct {
+		cl *CompoundList
+	}
+)
+
+func NewBraceGroup(cl Attrib) (*BraceGroup, error) {
+	return &BraceGroup{
+		cl: cl.(*CompoundList),
+	}, nil
+}
+
+// DoGroup
+
+type (
+	DoGroup struct {
+		cl *CompoundList
+	}
+)
+
+func NewDoGroup(cl Attrib) (*DoGroup, error) {
+	return &DoGroup{cl.(*CompoundList)}, nil
+}
+
+// SimpleCommand
+
+type (
+	SimpleCommand struct {
+		prefix *CmdPrefix
+		name   Name
+		word   Word
+		suffix *CmdSuffix
+	}
+)
+
+func NewSimpleCommand(prefix, name, word, suffix Attrib) (*SimpleCommand, error) {
+	return &SimpleCommand{
+		prefix: prefix.(*CmdPrefix),
+		name:   name.(Name),
+		word:   word.(Word),
+		suffix: suffix.(*CmdSuffix),
+	}, nil
+}
+
+// CmdName
+
+type (
+	CmdName Word
+)
+
+func NewCmdName(w Attrib) (CmdName, error) {
+	return CmdName(w.(Word)), nil
+}
+
+// CmdWord
+
+type (
+	CmdWord Word
+)
+
+func NewCmdWord(w Attrib) (CmdWord, error) {
+	return CmdWord(w.(Word)), nil
+}
+
+// CmdPrefix
+
+type (
+	CmdPrefix struct {
+		assign *Assignment
+		redir  *IoRedirect
+		prefix *CmdPrefix
+	}
+)
+
+func NewCmdPrefix(assign, redir, prefix Attrib) (*CmdPrefix, error) {
+	return &CmdPrefix{
+		assign: assign.(*Assignment),
+		redir:  assign.(*IoRedirect),
+		prefix: prefix.(*CmdPrefix),
+	}, nil
+}
+
+// CmdSuffix
+
+type (
+	CmdSuffix struct {
+		word   Word
+		redir  *IoRedirect
+		suffix *CmdSuffix
+	}
+)
+
+func NewCmdSuffix(word, redir, suffix Attrib) (*CmdSuffix, error) {
+	return &CmdSuffix{
+		word:   word.(Word),
+		redir:  redir.(*IoRedirect),
+		suffix: suffix.(*CmdSuffix),
+	}, nil
+}
+
+// RedirectList
+
+type (
+	RedirectList []*IoRedirect
+)
+
+func NewRedirectList(redir Attrib) (RedirectList, error) {
+	return RedirectList{redir.(*IoRedirect)}, nil
+}
+
+func AppendToRedirectList(list Attrib, el Attrib) (RedirectList, error) {
+	l := list.(RedirectList)
+	l = append(l, el.(*IoRedirect))
+	return l, nil
+}
+
+// IoRedirect
+
+type (
+	IoRedirect struct {
+		file *IoFile
+		num  IoNumber
+		here *IoHere
+	}
+)
+
+func NewIoRedirect(file Attrib, num Attrib, here Attrib) (*IoRedirect, error) {
+	return &IoRedirect{
+		file: file.(*IoFile),
+		num:  num.(IoNumber),
+		here: here.(*IoHere),
+	}, nil
+}
+
+// IoFile
+
+type (
+	Redirection int
+	IoFile      struct {
+		file  Filename
+		redir Redirection
+	}
+)
+
+const (
+	R_STDIN Redirection = iota
+	R_INFD
+	R_STDOUT
+	R_OUTFD
+	R_APPEND
+	R_ORWFD
+	R_OUTSP
+)
+
+func NewIoFile(file Attrib, redir Redirection) (*IoFile, error) {
+	return &IoFile{
+		file:  file.(Filename),
+		redir: redir,
+	}, nil
+}
+
+// Filename
+
+type (
+	Filename Word
+)
+
+func NewFilename(w Attrib) (Filename, error) {
+	return Filename(w.(Word)), nil
+}
+
+// IoHere
+
+type (
+	IoHere struct {
+		word         Word
+		suppressTabs bool
+	}
+)
+
+func NewIoHere(w Attrib, st bool) (*IoHere, error) {
+	return &IoHere{
+		word:         w.(Word),
+		suppressTabs: st,
+	}, nil
+}
+
+// HereEnd
+
+type (
+	HereEnd Word
+)
+
+func NewHereEnd(w Attrib) (HereEnd, error) {
+	return HereEnd(w.(Word)), nil
+}
+
+// NewlineList
+
+// Linebreak
+
+// SeparatorOp
+
+type (
+	SeparatorOp int
+)
+
+const (
+	S_AMP SeparatorOp = iota
+	S_SEMICOLON
+)
+
+// Separartor
+
+type (
+	Separator struct {
+		s SeparatorOp
+	}
+)
+
+func NewSeparator(op Attrib) (*Separator, error) {
+	return &Separator{op.(SeparatorOp)}, nil
+}
+
+// SequentialSep

errors/errors.go

@@ -0,0 +1,39 @@
+
+package errors
+
+import(
+	"bytes"
+	"fmt"
+	"github.com/localhots/penny/token"
+)
+
+type ErrorSymbol interface {
+}
+
+type Error struct {
+	Err            error
+	ErrorToken     *token.Token
+	ErrorSymbols   []ErrorSymbol
+	ExpectedTokens []string
+}
+
+func (E *Error) String() string {
+	w := new(bytes.Buffer)
+	fmt.Fprintf(w, "Error")
+	if E.Err != nil {
+		fmt.Fprintf(w, " %s\n", E.Err)
+	} else {
+		fmt.Fprintf(w, "\n")
+	}
+	fmt.Fprintf(w, "Token: type=%d, lit=%s\n", E.ErrorToken.Type, E.ErrorToken.Lit)
+	fmt.Fprintf(w, "Pos: offset=%d, line=%d, column=%d\n", E.ErrorToken.Pos.Offset, E.ErrorToken.Pos.Line, E.ErrorToken.Pos.Column)
+	fmt.Fprintf(w, "Expected one of: ")
+	for _, sym := range E.ExpectedTokens {
+		fmt.Fprintf(w, "%s ", sym)
+	}
+	fmt.Fprintf(w, "ErrorSymbol:\n")
+	for _, sym := range E.ErrorSymbols {
+		fmt.Fprintf(w, "%v\n", sym)
+	}
+	return w.String()
+}

lexer/acttab.go

@@ -0,0 +1,314 @@
+
+package lexer
+
+import(
+	"fmt"
+	"github.com/localhots/penny/token"
+)
+
+type ActionTable [NumStates] ActionRow
+
+type ActionRow struct {
+	Accept token.Type
+	Ignore string
+}
+
+func (this ActionRow) String() string {
+	return fmt.Sprintf("Accept=%d, Ignore=%s", this.Accept, this.Ignore)
+}
+
+var ActTab = ActionTable{
+ 	ActionRow{ // S0
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S1
+		Accept: -1,
+ 		Ignore: "!whitespace",
+ 	},
+ 	ActionRow{ // S2
+		Accept: 7,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S3
+		Accept: 39,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S4
+		Accept: 9,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S5
+		Accept: 10,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S6
+		Accept: 3,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S7
+		Accept: 40,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S8
+		Accept: 28,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S9
+		Accept: 4,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S10
+		Accept: 30,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S11
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S12
+		Accept: 13,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S13
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S14
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S15
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S16
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S17
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S18
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S19
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S20
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S21
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S22
+		Accept: 24,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S23
+		Accept: 8,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S24
+		Accept: 25,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S25
+		Accept: 38,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S26
+		Accept: 5,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S27
+		Accept: 16,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S28
+		Accept: 29,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S29
+		Accept: 35,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S30
+		Accept: 33,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S31
+		Accept: 31,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S32
+		Accept: 32,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S33
+		Accept: 34,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S34
+		Accept: 37,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S35
+		Accept: 13,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S36
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S37
+		Accept: 2,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S38
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S39
+		Accept: 26,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S40
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S41
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S42
+		Accept: 19,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S43
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S44
+		Accept: 17,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S45
+		Accept: 12,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S46
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S47
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S48
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S49
+		Accept: 6,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S50
+		Accept: 36,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S51
+		Accept: 13,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S52
+		Accept: 2,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S53
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S54
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S55
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S56
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S57
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S58
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S59
+		Accept: 11,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S60
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S61
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S62
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S63
+		Accept: 14,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S64
+		Accept: 27,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S65
+		Accept: 20,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S66
+		Accept: 21,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S67
+		Accept: 15,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S68
+		Accept: 18,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S69
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S70
+		Accept: 0,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S71
+		Accept: 23,
+ 		Ignore: "",
+ 	},
+ 	ActionRow{ // S72
+		Accept: 22,
+ 		Ignore: "",
+ 	},
+ 		
+}

lexer/lexer.go

@@ -0,0 +1,233 @@
+
+package lexer
+
+import (
+	
+	// "fmt"
+	// "github.com/localhots/penny/util"
+	
+	"io/ioutil"
+	"unicode/utf8"
+	"github.com/localhots/penny/token"
+)
+
+const(
+	NoState = -1
+	NumStates = 73
+	NumSymbols = 90
+) 
+
+type Lexer struct {
+	src             []byte
+	pos             int
+	line            int
+	column          int
+}
+
+func NewLexer(src []byte) *Lexer {
+	lexer := &Lexer{
+		src:    src,
+		pos:    0,
+		line:   1,
+		column: 1,
+	}
+	return lexer
+}
+
+func NewLexerFile(fpath string) (*Lexer, error) {
+	src, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return nil, err
+	}
+	return NewLexer(src), nil
+}
+
+func (this *Lexer) Scan() (tok *token.Token) {
+	
+	// fmt.Printf("Lexer.Scan() pos=%d\n", this.pos)
+	
+	tok = new(token.Token)
+	if this.pos >= len(this.src) {
+		tok.Type = token.EOF
+		tok.Pos.Offset, tok.Pos.Line, tok.Pos.Column = this.pos, this.line, this.column
+		return
+	}
+	start, end := this.pos, 0
+	tok.Type = token.INVALID
+	state, rune1, size := 0, rune(-1), 0
+	for state != -1 {
+	
+		// fmt.Printf("\tpos=%d, line=%d, col=%d, state=%d\n", this.pos, this.line, this.column, state)
+	
+		if this.pos >= len(this.src) {
+			rune1 = -1
+		} else {
+			rune1, size = utf8.DecodeRune(this.src[this.pos:])
+			this.pos += size
+		}
+		switch rune1 {
+		case '\n':
+			this.line++
+			this.column = 1
+		case '\r':
+			this.column = 1
+		case '\t':
+			this.column += 4
+		default:
+			this.column++
+		}
+
+	
+		// Production start
+		if rune1 != -1 {
+			state = TransTab[state](rune1)
+		} else {
+			state = -1
+		}
+		// Production end
+
+		// Debug start
+		// nextState := -1
+		// if rune1 != -1 {
+		// 	nextState = TransTab[state](rune1)
+		// }
+		// fmt.Printf("\tS%d, : tok=%s, rune == %s(%x), next state == %d\n", state, token.TokMap.Id(tok.Type), util.RuneToString(rune1), rune1, nextState)
+		// fmt.Printf("\t\tpos=%d, size=%d, start=%d, end=%d\n", this.pos, size, start, end)
+		// if nextState != -1 {
+		// 	fmt.Printf("\t\taction:%s\n", ActTab[nextState].String())
+		// }
+		// state = nextState
+		// Debug end
+	
+
+		if state != -1 {
+			switch {
+			case ActTab[state].Accept != -1:
+				tok.Type = ActTab[state].Accept
+				// fmt.Printf("\t Accept(%s), %s(%d)\n", string(act), token.TokMap.Id(tok), tok)
+				end = this.pos
+			case ActTab[state].Ignore != "":
+				// fmt.Printf("\t Ignore(%s)\n", string(act))
+				start = this.pos
+				state = 0
+				if start >= len(this.src) {
+					tok.Type = token.EOF
+				}
+
+			}
+		} else {
+			if tok.Type == token.INVALID {
+				end = this.pos
+			}
+		}
+	}
+	if end > start {
+		this.pos = end
+		tok.Lit = this.src[start:end]
+	} else {
+		tok.Lit = []byte{}
+	}
+	tok.Pos.Offset = start
+	tok.Pos.Column = this.column
+	tok.Pos.Line = this.line
+	return
+}
+
+func (this *Lexer) Reset() {
+	this.pos = 0
+}
+
+/*
+Lexer symbols:
+0: '_'
+1: '_'
+2: '_'
+3: '='
+4: '&'
+5: '&'
+6: '|'
+7: '|'
+8: '!'
+9: '|'
+10: '('
+11: ')'
+12: 'f'
+13: 'o'
+14: 'r'
+15: 'i'
+16: 'n'
+17: 'c'
+18: 'a'
+19: 's'
+20: 'e'
+21: 'e'
+22: 's'
+23: 'a'
+24: 'c'
+25: ';'
+26: ';'
+27: 'i'
+28: 'f'
+29: 't'
+30: 'h'
+31: 'e'
+32: 'n'
+33: 'f'
+34: 'i'
+35: 'e'
+36: 'l'
+37: 'i'
+38: 'f'
+39: 'e'
+40: 'l'
+41: 's'
+42: 'e'
+43: 'w'
+44: 'h'
+45: 'i'
+46: 'l'
+47: 'e'
+48: 'u'
+49: 'n'
+50: 't'
+51: 'i'
+52: 'l'
+53: '{'
+54: '}'
+55: 'd'
+56: 'o'
+57: 'd'
+58: 'o'
+59: 'n'
+60: 'e'
+61: '<'
+62: '<'
+63: '&'
+64: '>'
+65: '>'
+66: '&'
+67: '>'
+68: '>'
+69: '<'
+70: '>'
+71: '>'
+72: '|'
+73: '<'
+74: '<'
+75: '<'
+76: '<'
+77: '-'
+78: '\'
+79: 'n'
+80: '&'
+81: ';'
+82: ' '
+83: '\t'
+84: '\n'
+85: '\r'
+86: 'a'-'z'
+87: 'A'-'Z'
+88: '0'-'9'
+89: .
+
+*/

parser/action.go

@@ -0,0 +1,50 @@
+
+package parser
+
+import (
+	"fmt"
+)
+
+type action interface {
+	act()
+	String() string
+}
+
+type (
+	accept bool
+	shift  int // value is next state index
+	reduce int // value is production index
+)
+
+func (this accept) act() {}
+func (this shift) act()  {}
+func (this reduce) act() {}
+
+func (this accept) Equal(that action) bool {
+	if _, ok := that.(accept); ok {
+		return true
+	}
+	return false
+}
+
+func (this reduce) Equal(that action) bool {
+	that1, ok := that.(reduce)
+	if !ok {
+		return false
+	}
+	return this == that1
+}
+
+func (this shift) Equal(that action) bool {
+	that1, ok := that.(shift)
+	if !ok {
+		return false
+	}
+	return this == that1
+}
+
+func (this accept) String() string { return "accept(0)" }
+func (this shift) String() string  { return fmt.Sprintf("shift:%d", this) }
+func (this reduce) String() string {
+	return fmt.Sprintf("reduce:%d(%s)", this, productionsTable[this].String)
+}

parser/actiontable.go

@@ -0,0 +1,159 @@
+
+package parser
+
+type(
+	actionTable [numStates]actionRow
+	actionRow struct {
+		canRecover bool
+		actions [numSymbols]action
+	}
+)
+
+var actionTab = actionTable{
+	actionRow{ // S0
+				canRecover: false,
+		actions: [numSymbols]action{
+			nil,		/* INVALID */
+			nil,		/* $ */
+			shift(2),		/* word */
+			nil,		/* number */
+			nil,		/* = */
+			nil,		/* && */
+			nil,		/* || */
+			nil,		/* ! */
+			nil,		/* | */
+			nil,		/* ( */
+			nil,		/* ) */
+			nil,		/* for */
+			nil,		/* in */
+			nil,		/* name */
+			nil,		/* case */
+			nil,		/* esac */
+			nil,		/* ;; */
+			nil,		/* if */
+			nil,		/* then */
+			nil,		/* fi */
+			nil,		/* elif */
+			nil,		/* else */
+			nil,		/* while */
+			nil,		/* until */
+			nil,		/* { */
+			nil,		/* } */
+			nil,		/* do */
+			nil,		/* done */
+			nil,		/* < */
+			nil,		/* <& */
+			nil,		/* > */
+			nil,		/* >& */
+			nil,		/* >> */
+			nil,		/* <> */
+			nil,		/* >| */
+			nil,		/* << */
+			nil,		/* <<- */
+			nil,		/* \n */
+			nil,		/* nothing */
+			nil,		/* & */
+			nil,		/* ; */
+			
+		},
+
+	},
+	actionRow{ // S1
+				canRecover: false,
+		actions: [numSymbols]action{
+			nil,		/* INVALID */
+			accept(true),		/* $ */
+			nil,		/* word */
+			nil,		/* number */
+			nil,		/* = */
+			nil,		/* && */
+			nil,		/* || */
+			nil,		/* ! */
+			nil,		/* | */
+			nil,		/* ( */
+			nil,		/* ) */
+			nil,		/* for */
+			nil,		/* in */
+			nil,		/* name */
+			nil,		/* case */
+			nil,		/* esac */
+			nil,		/* ;; */
+			nil,		/* if */
+			nil,		/* then */
+			nil,		/* fi */
+			nil,		/* elif */
+			nil,		/* else */
+			nil,		/* while */
+			nil,		/* until */
+			nil,		/* { */
+			nil,		/* } */
+			nil,		/* do */
+			nil,		/* done */
+			nil,		/* < */
+			nil,		/* <& */
+			nil,		/* > */
+			nil,		/* >& */
+			nil,		/* >> */
+			nil,		/* <> */
+			nil,		/* >| */
+			nil,		/* << */
+			nil,		/* <<- */
+			nil,		/* \n */
+			nil,		/* nothing */
+			nil,		/* & */
+			nil,		/* ; */
+			
+		},
+
+	},
+	actionRow{ // S2
+				canRecover: false,
+		actions: [numSymbols]action{
+			nil,		/* INVALID */
+			reduce(1),		/* $, reduce: Word */
+			nil,		/* word */
+			nil,		/* number */
+			nil,		/* = */
+			nil,		/* && */
+			nil,		/* || */
+			nil,		/* ! */
+			nil,		/* | */
+			nil,		/* ( */
+			nil,		/* ) */
+			nil,		/* for */
+			nil,		/* in */
+			nil,		/* name */
+			nil,		/* case */
+			nil,		/* esac */
+			nil,		/* ;; */
+			nil,		/* if */
+			nil,		/* then */
+			nil,		/* fi */
+			nil,		/* elif */
+			nil,		/* else */
+			nil,		/* while */
+			nil,		/* until */
+			nil,		/* { */
+			nil,		/* } */
+			nil,		/* do */
+			nil,		/* done */
+			nil,		/* < */
+			nil,		/* <& */
+			nil,		/* > */
+			nil,		/* >& */
+			nil,		/* >> */
+			nil,		/* <> */
+			nil,		/* >| */
+			nil,		/* << */
+			nil,		/* <<- */
+			nil,		/* \n */
+			nil,		/* nothing */
+			nil,		/* & */
+			nil,		/* ; */
+			
+		},
+
+	},
+	
+}
+

parser/gototable.go

@@ -0,0 +1,170 @@
+
+/*
+*/
+package parser
+
+const numNTSymbols = 47
+type(
+	gotoTable [numStates]gotoRow
+	gotoRow	[numNTSymbols] int
+)
+
+var gotoTab = gotoTable{
+	gotoRow{ // S0
+		
+		-1, // S'
+		1, // Word
+		-1, // IoNumber
+		-1, // AssignmentWord
+		-1, // List
+		-1, // AndOr
+		-1, // Pipeline
+		-1, // PipeSequence
+		-1, // Command
+		-1, // CompoundCommand
+		-1, // Subshell
+		-1, // CompoundList
+		-1, // Term
+		-1, // ForClause
+		-1, // Name
+		-1, // Wordlist
+		-1, // CaseClause
+		-1, // CaseListNs
+		-1, // CaseList
+		-1, // CaseItemNs
+		-1, // CaseItem
+		-1, // Pattern
+		-1, // IfClause
+		-1, // ElsePart
+		-1, // WhileClause
+		-1, // UntilClause
+		-1, // FunctionDefinition
+		-1, // FunctionBody
+		-1, // FunctionName
+		-1, // BraceGroup
+		-1, // DoGroup
+		-1, // SimpleCommand
+		-1, // CmdName
+		-1, // CmdWord
+		-1, // CmdPrefix
+		-1, // CmdSuffix
+		-1, // RedirectList
+		-1, // IoRedirect
+		-1, // IoFile
+		-1, // Filename
+		-1, // IoHere
+		-1, // HereEnd
+		-1, // NewlineList
+		-1, // Linebreak
+		-1, // SeparatorOp
+		-1, // Separator
+		-1, // SequentialSep
+		
+
+	},
+	gotoRow{ // S1
+		
+		-1, // S'
+		-1, // Word
+		-1, // IoNumber
+		-1, // AssignmentWord
+		-1, // List
+		-1, // AndOr
+		-1, // Pipeline
+		-1, // PipeSequence
+		-1, // Command
+		-1, // CompoundCommand
+		-1, // Subshell
+		-1, // CompoundList
+		-1, // Term
+		-1, // ForClause
+		-1, // Name
+		-1, // Wordlist
+		-1, // CaseClause
+		-1, // CaseListNs
+		-1, // CaseList
+		-1, // CaseItemNs
+		-1, // CaseItem
+		-1, // Pattern
+		-1, // IfClause
+		-1, // ElsePart
+		-1, // WhileClause
+		-1, // UntilClause
+		-1, // FunctionDefinition
+		-1, // FunctionBody
+		-1, // FunctionName
+		-1, // BraceGroup
+		-1, // DoGroup
+		-1, // SimpleCommand
+		-1, // CmdName
+		-1, // CmdWord
+		-1, // CmdPrefix
+		-1, // CmdSuffix
+		-1, // RedirectList
+		-1, // IoRedirect
+		-1, // IoFile
+		-1, // Filename
+		-1, // IoHere
+		-1, // HereEnd
+		-1, // NewlineList
+		-1, // Linebreak
+		-1, // SeparatorOp
+		-1, // Separator
+		-1, // SequentialSep
+		
+
+	},
+	gotoRow{ // S2
+		
+		-1, // S'
+		-1, // Word
+		-1, // IoNumber
+		-1, // AssignmentWord
+		-1, // List
+		-1, // AndOr
+		-1, // Pipeline
+		-1, // PipeSequence
+		-1, // Command
+		-1, // CompoundCommand
+		-1, // Subshell
+		-1, // CompoundList
+		-1, // Term
+		-1, // ForClause
+		-1, // Name
+		-1, // Wordlist
+		-1, // CaseClause
+		-1, // CaseListNs
+		-1, // CaseList
+		-1, // CaseItemNs
+		-1, // CaseItem
+		-1, // Pattern
+		-1, // IfClause
+		-1, // ElsePart
+		-1, // WhileClause
+		-1, // UntilClause
+		-1, // FunctionDefinition
+		-1, // FunctionBody
+		-1, // FunctionName
+		-1, // BraceGroup
+		-1, // DoGroup
+		-1, // SimpleCommand
+		-1, // CmdName
+		-1, // CmdWord
+		-1, // CmdPrefix
+		-1, // CmdSuffix
+		-1, // RedirectList
+		-1, // IoRedirect
+		-1, // IoFile
+		-1, // Filename
+		-1, // IoHere
+		-1, // HereEnd
+		-1, // NewlineList
+		-1, // Linebreak
+		-1, // SeparatorOp
+		-1, // Separator
+		-1, // SequentialSep
+		
+
+	},
+	
+}

parser/parser.go

@@ -0,0 +1,214 @@
+
+package parser
+
+import(
+	"bytes"
+	"fmt"
+	"errors"
+	parseError "github.com/localhots/penny/errors"
+	"github.com/localhots/penny/token"
+)
+
+const (
+	numProductions = 108
+	numStates      = 3
+	numSymbols     = 88
+)
+
+// Stack
+
+type stack struct {
+	state []int
+	attrib	[]Attrib
+}
+
+const iNITIAL_STACK_SIZE = 100
+
+func newStack() *stack {
+	return &stack{ 	state: 	make([]int, 0, iNITIAL_STACK_SIZE),
+					attrib: make([]Attrib, 0, iNITIAL_STACK_SIZE),
+			}
+}
+
+func (this *stack) reset() {
+	this.state = this.state[0:0]
+	this.attrib = this.attrib[0:0]
+}
+
+func (this *stack) push(s int, a Attrib) {
+	this.state = append(this.state, s)
+	this.attrib = append(this.attrib, a)
+}
+
+func(this *stack) top() int {
+	return this.state[len(this.state) - 1]
+}
+
+func (this *stack) peek(pos int) int {
+	return this.state[pos]
+}
+
+func (this *stack) topIndex() int {
+	return len(this.state) - 1
+}
+
+func (this *stack) popN(items int) []Attrib {
+	lo, hi := len(this.state) - items, len(this.state)
+	
+	attrib := this.attrib[lo: hi]
+	
+	this.state = this.state[:lo]
+	this.attrib = this.attrib[:lo]
+	
+	return attrib
+}
+
+func (S *stack) String() string {
+	w := new(bytes.Buffer)
+	fmt.Fprintf(w, "stack:\n")
+	for i, st := range S.state {
+		fmt.Fprintf(w, "\t%d:%d , ", i, st)
+		if S.attrib[i] == nil {
+			fmt.Fprintf(w, "nil")
+		} else {
+			fmt.Fprintf(w, "%v", S.attrib[i])
+		}
+		w.WriteString("\n")
+	}
+	return w.String()
+}
+
+// Parser
+
+type Parser struct {
+	stack     *stack
+	nextToken *token.Token
+	pos       int
+}
+
+type Scanner interface {
+	Scan() (tok *token.Token)
+}
+
+func NewParser() *Parser {
+	p := &Parser{stack: newStack()}
+	p.Reset()
+	return p
+}
+
+func (P *Parser) Reset() {
+	P.stack.reset()
+	P.stack.push(0, nil)
+}
+
+func (P *Parser) Error(err error, scanner Scanner) (recovered bool, errorAttrib *parseError.Error) {
+	errorAttrib = &parseError.Error{
+		Err:            err,
+		ErrorToken:     P.nextToken,
+		ErrorSymbols:   P.popNonRecoveryStates(),
+		ExpectedTokens: make([]string, 0, 8),
+	}
+	for t, action := range actionTab[P.stack.top()].actions {
+		if action != nil {
+			errorAttrib.ExpectedTokens = append(errorAttrib.ExpectedTokens, token.TokMap.Id(token.Type(t)))
+		}
+	}
+
+	if action := actionTab[P.stack.top()].actions[token.TokMap.Type("error")]; action != nil {
+		P.stack.push(int(action.(shift)), errorAttrib) // action can only be shift
+	} else {
+		return
+	}
+
+	if action := actionTab[P.stack.top()].actions[P.nextToken.Type]; action != nil {
+		recovered = true
+	}
+	for !recovered && P.nextToken.Type != token.EOF {
+		P.nextToken = scanner.Scan()
+		if action := actionTab[P.stack.top()].actions[P.nextToken.Type]; action != nil {
+			recovered = true
+		}
+	}
+
+	return
+}
+
+func (P *Parser) popNonRecoveryStates() (removedAttribs []parseError.ErrorSymbol) {
+	if rs, ok := P.firstRecoveryState(); ok {
+		errorSymbols := P.stack.popN(int(P.stack.topIndex() - rs))
+		removedAttribs = make([]parseError.ErrorSymbol, len(errorSymbols))
+		for i, e := range errorSymbols {
+			removedAttribs[i] = e
+		}
+	} else {
+		removedAttribs = []parseError.ErrorSymbol{}
+	}
+	return
+}
+
+// recoveryState points to the highest state on the stack, which can recover
+func (P *Parser) firstRecoveryState() (recoveryState int, canRecover bool) {
+	recoveryState, canRecover = P.stack.topIndex(), actionTab[P.stack.top()].canRecover
+	for recoveryState > 0 && !canRecover {
+		recoveryState--
+		canRecover = actionTab[P.stack.peek(recoveryState)].canRecover
+	}
+	return
+}
+
+func (P *Parser) newError(err error) error {
+	w := new(bytes.Buffer)
+	fmt.Fprintf(w, "Error in S%d: %s, %s", P.stack.top(), token.TokMap.TokenString(P.nextToken), P.nextToken.Pos.String())
+	if err != nil {
+		w.WriteString(err.Error())
+	} else {
+		w.WriteString(", expected one of: ")
+		actRow := actionTab[P.stack.top()]
+		for i, t := range actRow.actions {
+			if t != nil {
+				fmt.Fprintf(w, "%s ", token.TokMap.Id(token.Type(i)))
+			}
+		}
+	}
+	return errors.New(w.String())
+}
+
+func (this *Parser) Parse(scanner Scanner) (res interface{}, err error) {
+	this.Reset()
+	this.nextToken = scanner.Scan()
+	for acc := false; !acc; {
+		action := actionTab[this.stack.top()].actions[this.nextToken.Type]
+		if action == nil {
+			if recovered, errAttrib := this.Error(nil, scanner); !recovered {
+				this.nextToken = errAttrib.ErrorToken
+				return nil, this.newError(nil)
+			}
+			if action = actionTab[this.stack.top()].actions[this.nextToken.Type]; action == nil {
+				panic("Error recovery led to invalid action")
+			}
+		}
+		
+		// fmt.Printf("S%d %s %s\n", this.stack.top(), token.TokMap.TokenString(this.nextToken), action.String())
+		
+
+		switch act := action.(type) {
+		case accept:
+			res = this.stack.popN(1)[0]
+			acc = true
+		case shift:
+			this.stack.push(int(act), this.nextToken)
+			this.nextToken = scanner.Scan()
+		case reduce:
+			prod := productionsTable[int(act)]
+			attrib, err := prod.ReduceFunc(this.stack.popN(prod.NumSymbols))
+			if err != nil {
+				return nil, this.newError(err)
+			} else {
+				this.stack.push(gotoTab[this.stack.top()][prod.NTType], attrib)
+			}
+		default:
+			panic("unknown action: " + action.String())
+		}
+	}
+	return res, nil
+}

sh.bnf

@@ -0,0 +1,223 @@
+!whitespace : ' ' | '\t' | '\n' | '\r' ;
+
+_letter : 'a'-'z' 'A'-'Z' ;
+_digit : '0'-'9' ;
+
+number : _digit {_digit} ;
+word /*BAD*/ : _letter {_letter | '_'} ;
+nothing : . ;
+
+name : ( _letter | '_' ) { _letter | '_' | _digit } ;
+
+<< import "github.com/localhots/penny/ast" >>
+
+/* Tokens */
+
+Word
+    : word << ast.NewWord($0) >>
+    ;
+IoNumber
+    : number << ast.NewIoNumber($0) >>
+    ;
+
+/* Commands */
+
+/*
+CompleteCommand
+    : List Separator
+    | List
+    ;
+*/
+AssignmentWord
+    : Word "=" Word << ast.NewAssignment($0, $2) >>
+    ;
+List
+    : List SeparatorOp AndOr << ast.AppendToList($0, $2) >>
+    | AndOr                  << ast.NewList($0) >>
+    ;
+AndOr
+    : Pipeline                      << ast.NewAndOr($0) >>
+    | AndOr "&&" Linebreak Pipeline << ast.AppendAnd($0, $3) >>
+    | AndOr "||" Linebreak Pipeline << ast.AppendOr($0, $3) >>
+    ;
+Pipeline
+    : PipeSequence     << ast.NewPipeline($1, false) >>
+    | "!" PipeSequence << ast.NewPipeline($1, true) >>
+    ;
+PipeSequence
+    : Command                            << ast.NewPipeSequence($0) >>
+    | PipeSequence "|" Linebreak Command << ast.AppendToPipeSequence($0, $3) >>
+    ;
+Command
+    : SimpleCommand                << ast.NewCommand($0, nil) >>
+    | CompoundCommand              << ast.NewCommand($0, nil) >>
+    | CompoundCommand RedirectList << ast.NewCommand($0, $1) >>
+    | FunctionDefinition           << ast.NewCommand($0, nil) >>
+    ;
+CompoundCommand
+    : BraceGroup  << ast.NewCompoundCommand($0) >>
+    | Subshell    << ast.NewCompoundCommand($0) >>
+    | ForClause   << ast.NewCompoundCommand($0) >>
+    | CaseClause  << ast.NewCompoundCommand($0) >>
+    | IfClause    << ast.NewCompoundCommand($0) >>
+    | WhileClause << ast.NewCompoundCommand($0) >>
+    | UntilClause << ast.NewCompoundCommand($0) >>
+    ;
+Subshell
+    : "(" CompoundList ")" << ast.NewSubshell($1) >>
+    ;
+CompoundList
+    : Term                        << ast.NewCompoundList($0, nil) >>
+    | NewlineList Term            << ast.NewCompoundList($1, $2) >>
+    | Term Separator              << ast.NewCompoundList($0, $1) >>
+    | NewlineList Term Separator  << ast.NewCompoundList($1, $2) >>
+    ;
+Term
+    : Term Separator AndOr << ast.AppendToTerm($0, $2, $1) >>
+    | AndOr                << ast.NewTerm($0) >>
+    ;
+ForClause
+    : "for" Name Linebreak DoGroup                             << ast.NewForClause($1, ast.Wordlist{}, $3) >>
+    | "for" Name Linebreak "in" SequentialSep DoGroup          << ast.NewForClause($1, ast.Wordlist{}, $5) >>
+    | "for" Name Linebreak "in" Wordlist SequentialSep DoGroup << ast.NewForClause($1, $4, $6) >>
+    ;
+Name
+    : name << ast.NewName($0) >> /* Apply rule 5 */
+    ;
+Wordlist
+    : Wordlist Word << ast.AppendToWordlist($0, $1) >>
+    | Word          << ast.NewWordlist($0) >>
+    ;
+CaseClause
+    : "case" Word Linebreak "in" Linebreak CaseList "esac"   << ast.NewCaseClause($1, $5) >>
+    | "case" Word Linebreak "in" Linebreak CaseListNs "esac" << ast.NewCaseClause($1, $5) >>
+    | "case" Word Linebreak "in" Linebreak "esac"            << ast.NewCaseClause($1, ast.CaseList{}) >>
+    ;
+CaseListNs
+    : CaseList CaseItemNs << ast.AppendToCaseList($0, $1) >>
+    | CaseItemNs          << ast.NewCaseList($0) >>
+    ;
+CaseList
+    : CaseList CaseItem  << ast.AppendToCaseList($0, $1) >>
+    | CaseItem           << ast.NewCaseList($0) >>
+    ;
+CaseItemNs
+    : Pattern ")" Linebreak                  << ast.NewCaseItem($0, nil) >>
+    | Pattern ")" CompoundList Linebreak     << ast.NewCaseItem($0, $2) >>
+    | "(" Pattern ")" Linebreak              << ast.NewCaseItem($1, nil) >>
+    | "(" Pattern ")" CompoundList Linebreak << ast.NewCaseItem($1, $3) >>
+    ;
+CaseItem
+    : Pattern ")" Linebreak ";;" Linebreak        << ast.NewCaseItem($0, nil) >>
+    | Pattern ")" CompoundList ";;" Linebreak     << ast.NewCaseItem($0, $2) >>
+    | "(" Pattern ")" Linebreak ";;" Linebreak    << ast.NewCaseItem($1, nil) >>
+    | "(" Pattern ")" CompoundList ";;" Linebreak << ast.NewCaseItem($1, $3) >>
+    ;
+Pattern
+    : Word             << ast.NewPattern($0) >>  /* Apply rule 4 */
+    | Pattern "|" Word << ast.AppendToPattern($0, $2) >> /* Do not apply rule 4 */
+    ;
+IfClause
+    : "if" CompoundList "then" CompoundList ElsePart "fi" << ast.NewIfClause($1, $3, $4) >>
+    | "if" CompoundList "then" CompoundList "fi"          << ast.NewIfClause($1, $3, nil) >>
+    ;
+ElsePart
+    : "elif" CompoundList "then" ElsePart << ast.NewIfClause($1, nil, $3) >>
+    | "else" CompoundList                 << ast.NewIfClause(nil, $1, nil) >>
+    ;
+WhileClause
+    : "while" CompoundList DoGroup << ast.NewWhileClause($1, $2) >>
+    ;
+UntilClause
+    : "until" CompoundList DoGroup << ast.NewUntilClause($1, $2) >>
+    ;
+FunctionDefinition
+    : FunctionName "(" ")" Linebreak FunctionBody << ast.NewFunctionDefinition($0, $4) >>
+    ;
+FunctionBody
+    : CompoundCommand              << ast.NewFunctionBody($0, ast.RedirectList{}) >> /* Apply rule 9 */
+    | CompoundCommand RedirectList << ast.NewFunctionBody($0, $1) >> /* Apply rule 9 */
+    ;
+FunctionName
+    : Word << ast.NewFunctionName($0) >> /* Apply rule 8 */
+    ;
+BraceGroup
+    : "{" CompoundList "}" << ast.NewBraceGroup($1) >>
+    ;
+DoGroup
+    : "do" CompoundList "done" << ast.NewDoGroup($1) >> /* Apply rule 6 */
+    ;
+SimpleCommand
+    : CmdPrefix CmdWord CmdSuffix << ast.NewSimpleCommand($0,  nil, $1,  $2) >>
+    | CmdPrefix CmdWord           << ast.NewSimpleCommand($0,  nil, $1,  nil) >>
+    | CmdPrefix                   << ast.NewSimpleCommand($0,  nil, nil, nil) >>
+    | CmdName CmdSuffix           << ast.NewSimpleCommand(nil, $0,  nil, $1) >>
+    | CmdName                     << ast.NewSimpleCommand(nil, $0,  nil, nil) >>
+    ;
+CmdName
+    : Word << ast.NewCmdName($0) >> /* Apply rule 7a */
+    ;
+CmdWord
+    : Word << ast.NewCmdWord($0) >> /* Apply rule 7b */
+    ;
+CmdPrefix
+    : IoRedirect               << ast.NewCmdPrefix(nil, $0, nil) >>
+    | CmdPrefix IoRedirect     << ast.NewCmdPrefix(nil, $1, $0) >>
+    | AssignmentWord           << ast.NewCmdPrefix($0, nil, nil) >>
+    | CmdPrefix AssignmentWord << ast.NewCmdPrefix($1, nil, $0) >>
+    ;
+CmdSuffix
+    : IoRedirect           << ast.NewCmdSuffix(ast.Word(""), $0, nil) >>
+    | CmdSuffix IoRedirect << ast.NewCmdSuffix(ast.Word(""), $1, $0) >>
+    | Word                 << ast.NewCmdSuffix($0, nil, nil) >>
+    | CmdSuffix Word       << ast.NewCmdSuffix($1, nil, $0) >>
+    ;
+RedirectList
+    : IoRedirect              << ast.NewRedirectList($0) >>
+    | RedirectList IoRedirect << ast.AppendToRedirectList($0, $1) >>
+    ;
+IoRedirect
+    : IoFile          << ast.NewIoRedirect($0, ast.IoNumber(0), nil) >>
+    | IoNumber IoFile << ast.NewIoRedirect($1, $0, nil) >>
+    | IoHere          << ast.NewIoRedirect(nil, ast.IoNumber(0), $0) >>
+    | IoNumber IoHere << ast.NewIoRedirect(nil, $0, $1) >>
+    ;
+IoFile
+    : "<"  Filename << ast.NewIoFile($1, ast.R_STDIN) >>
+    | "<&" Filename << ast.NewIoFile($1, ast.R_INFD) >>
+    | ">"  Filename << ast.NewIoFile($1, ast.R_STDOUT) >>
+    | ">&" Filename << ast.NewIoFile($1, ast.R_OUTFD) >>
+    | ">>" Filename << ast.NewIoFile($1, ast.R_APPEND) >>
+    | "<>" Filename << ast.NewIoFile($1, ast.R_ORWFD) >>
+    | ">|" Filename << ast.NewIoFile($1, ast.R_OUTSP) >>
+    ;
+Filename
+    : Word << ast.NewFilename($0) >> /* Apply rule 2 */
+    ;
+IoHere
+    : "<<"  HereEnd << ast.NewIoHere($1, false) >>
+    | "<<-" HereEnd << ast.NewIoHere($1, true) >>
+    ;
+HereEnd
+    : Word << ast.NewHereEnd($0) >> /* Apply rule 3 */
+    ;
+NewlineList
+    : "\n"
+    | NewlineList "\n"
+    ;
+Linebreak
+    : NewlineList
+    | nothing
+    ;
+SeparatorOp
+    : "&" << ast.S_AMP, nil >>
+    | ";" << ast.S_SEMICOLON, nil >>
+    ;
+Separator
+    : SeparatorOp Linebreak << ast.NewSeparator($0) >>
+    | NewlineList
+    ;
+SequentialSep
+    : ";" Linebreak
+    | NewlineList
+    ;

test/case.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+case $1 in
+    foo) echo "FOOOOO!" ;;
+    bar) echo "BAAAAAR!" ;;
+    baz) echo "BAAAZZZ!" ;;
+    *) echo "WHATEVER!"
+esac

test/example.sh

@@ -0,0 +1,8 @@
+for i in 0..3
+do
+    echo "i = $i"
+done
+
+if [[ true ]]; then
+    echo "Truth!"
+fi

test/parse_test.go

@@ -0,0 +1,26 @@
+package test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/kr/pretty"
+	"github.com/localhots/penny/lexer"
+	"github.com/localhots/penny/parser"
+)
+
+func TestWorld(t *testing.T) {
+	b, err := ioutil.ReadFile("example.sh")
+	if err != nil {
+		panic(err)
+	}
+
+	lex := lexer.NewLexer(b)
+	p := parser.NewParser()
+	st, err := p.Parse(lex)
+	if err != nil {
+		panic(err)
+	}
+
+	pretty.Println(st)
+}

token/token.go

@@ -0,0 +1,148 @@
+
+package token
+
+import(
+	"fmt"
+)
+
+type Token struct {
+	Type
+	Lit []byte
+	Pos
+}
+
+type Type int
+
+const(
+	INVALID Type = iota
+	EOF
+)
+
+type Pos struct {
+	Offset int
+	Line int
+	Column int
+}
+
+func (this Pos) String() string {
+	return fmt.Sprintf("Pos(offset=%d, line=%d, column=%d)", this.Offset, this.Line, this.Column)
+}
+
+type TokenMap struct {
+	typeMap  []string
+	idMap map[string]Type
+}
+
+func (this TokenMap) Id(tok Type) string {
+	if int(tok) < len(this.typeMap) {
+		return this.typeMap[tok]
+	}
+	return "unknown"
+}
+
+func (this TokenMap) Type(tok string) Type {
+	if typ, exist := this.idMap[tok]; exist {
+		return typ
+	}
+	return INVALID
+}
+
+func (this TokenMap) TokenString(tok *Token) string {
+	//TODO: refactor to print pos & token string properly
+	return fmt.Sprintf("%s(%d,%s)", this.Id(tok.Type), tok.Type, tok.Lit)
+}
+
+func (this TokenMap) StringType(typ Type) string {
+	return fmt.Sprintf("%s(%d)", this.Id(typ), typ)
+}
+
+var TokMap = TokenMap{
+	typeMap: []string{
+		"INVALID",
+		"$",
+		"word",
+		"number",
+		"=",
+		"&&",
+		"||",
+		"!",
+		"|",
+		"(",
+		")",
+		"for",
+		"in",
+		"name",
+		"case",
+		"esac",
+		";;",
+		"if",
+		"then",
+		"fi",
+		"elif",
+		"else",
+		"while",
+		"until",
+		"{",
+		"}",
+		"do",
+		"done",
+		"<",
+		"<&",
+		">",
+		">&",
+		">>",
+		"<>",
+		">|",
+		"<<",
+		"<<-",
+		"\n",
+		"nothing",
+		"&",
+		";",
+	},
+
+	idMap: map[string]Type {
+		"INVALID": 0,
+		"$": 1,
+		"word": 2,
+		"number": 3,
+		"=": 4,
+		"&&": 5,
+		"||": 6,
+		"!": 7,
+		"|": 8,
+		"(": 9,
+		")": 10,
+		"for": 11,
+		"in": 12,
+		"name": 13,
+		"case": 14,
+		"esac": 15,
+		";;": 16,
+		"if": 17,
+		"then": 18,
+		"fi": 19,
+		"elif": 20,
+		"else": 21,
+		"while": 22,
+		"until": 23,
+		"{": 24,
+		"}": 25,
+		"do": 26,
+		"done": 27,
+		"<": 28,
+		"<&": 29,
+		">": 30,
+		">&": 31,
+		">>": 32,
+		"<>": 33,
+		">|": 34,
+		"<<": 35,
+		"<<-": 36,
+		"\n": 37,
+		"nothing": 38,
+		"&": 39,
+		";": 40,
+	},
+}
+

util/litconv.go

@@ -0,0 +1,121 @@
+
+//Copyright 2013 Vastech SA (PTY) LTD
+//
+//   Licensed under the Apache License, Version 2.0 (the "License");
+//   you may not use this file except in compliance with the License.
+//   You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//   Unless required by applicable law or agreed to in writing, software
+//   distributed under the License is distributed on an "AS IS" BASIS,
+//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//   See the License for the specific language governing permissions and
+//   limitations under the License.
+
+package util
+
+import (
+	"fmt"
+	"strconv"
+	"unicode"
+	"unicode/utf8"
+)
+
+/* Interface */
+
+/*
+Convert the literal value of a scanned token to rune
+*/
+func RuneValue(lit []byte) rune {
+	if lit[1] == '\\' {
+		return escapeCharVal(lit)
+	}
+	r, size := utf8.DecodeRune(lit[1:])
+	if size != len(lit)-2 {
+		panic(fmt.Sprintf("Error decoding rune. Lit: %s, rune: %d, size%d\n", lit, r, size))
+	}
+	return r
+}
+
+/*
+Convert the literal value of a scanned token to int64
+*/
+func IntValue(lit []byte) (int64, error) {
+	return strconv.ParseInt(string(lit), 10, 64)
+}
+
+/*
+Convert the literal value of a scanned token to uint64
+*/
+func UintValue(lit []byte) (uint64, error) {
+	return strconv.ParseUint(string(lit), 10, 64)
+}
+
+/* Util */
+
+func escapeCharVal(lit []byte) rune {
+	var i, base, max uint32
+	offset := 2
+	switch lit[offset] {
+	case 'a':
+		return '\a'
+	case 'b':
+		return '\b'
+	case 'f':
+		return '\f'
+	case 'n':
+		return '\n'
+	case 'r':
+		return '\r'
+	case 't':
+		return '\t'
+	case 'v':
+		return '\v'
+	case '\\':
+		return '\\'
+	case '\'':
+		return '\''
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		i, base, max = 3, 8, 255
+	case 'x':
+		i, base, max = 2, 16, 255
+		offset++
+	case 'u':
+		i, base, max = 4, 16, unicode.MaxRune
+		offset++
+	case 'U':
+		i, base, max = 8, 16, unicode.MaxRune
+		offset++
+	default:
+		panic(fmt.Sprintf("Error decoding character literal: %s\n", lit))
+	}
+
+	var x uint32
+	for ; i > 0 && offset < len(lit)-1; i-- {
+		ch, size := utf8.DecodeRune(lit[offset:])
+		offset += size
+		d := uint32(digitVal(ch))
+		if d >= base {
+			panic(fmt.Sprintf("charVal(%s): illegal character (%c) in escape sequence. size=%d, offset=%d", lit, ch, size, offset))
+		}
+		x = x*base + d
+	}
+	if x > max || 0xD800 <= x && x < 0xE000 {
+		panic(fmt.Sprintf("Error decoding escape char value. Lit:%s, offset:%d, escape sequence is invalid Unicode code point\n", lit, offset))
+	}
+
+	return rune(x)
+}
+
+func digitVal(ch rune) int {
+	switch {
+	case '0' <= ch && ch <= '9':
+		return int(ch) - '0'
+	case 'a' <= ch && ch <= 'f':
+		return int(ch) - 'a' + 10
+	case 'A' <= ch && ch <= 'F':
+		return int(ch) - 'A' + 10
+	}
+	return 16 // larger than any legal digit val
+}

util/rune.go

@@ -0,0 +1,52 @@
+
+//Copyright 2013 Vastech SA (PTY) LTD
+//
+//   Licensed under the Apache License, Version 2.0 (the "License");
+//   you may not use this file except in compliance with the License.
+//   You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+//   Unless required by applicable law or agreed to in writing, software
+//   distributed under the License is distributed on an "AS IS" BASIS,
+//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//   See the License for the specific language governing permissions and
+//   limitations under the License.
+
+package util
+
+import (
+	"fmt"
+)
+
+func RuneToString(r rune) string {
+	if r >= 0x20 && r < 0x7f {
+		return fmt.Sprintf("'%c'", r)
+	}
+	switch r {
+	case 0x07:
+		return "'\\a'"
+	case 0x08:
+		return "'\\b'"
+	case 0x0C:
+		return "'\\f'"
+	case 0x0A:
+		return "'\\n'"
+	case 0x0D:
+		return "'\\r'"
+	case 0x09:
+		return "'\\t'"
+	case 0x0b:
+		return "'\\v'"
+	case 0x5c:
+		return "'\\\\\\'"
+	case 0x27:
+		return "'\\''"
+	case 0x22:
+		return "'\\\"'"
+	}
+	if r < 0x10000 {
+		return fmt.Sprintf("\\u%04x", r)
+	}
+	return fmt.Sprintf("\\U%08x", r)
+}