qbit/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-11-22 00:04:41 -07:00
Code Releases Activity

Preparation for moving scanner into a lib:

Browse Source 
- separated out token definition from scanner
- cleaned up token and scanner implementation
- added comments to exported interfaces

R=r
OCL=25665
CL=25665
This commit is contained in:
Robert Griesemer 2009-03-03 18:25:07 -08:00
parent 337ce2220f
commit 77567265a8
7 changed files with 701 additions and 624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
usr/gri/pretty/Makefile
View File

@ -32,21 +32,21 @@ gds.6: utils.6 platform.6 compilation.6 printer.6
pretty.6: platform.6 printer.6 compilation.6
compilation.6: platform.6 scanner.6 parser.6 ast.6 typechecker.6
compilation.6: platform.6 token.6 scanner.6 parser.6 ast.6 typechecker.6
typechecker.6: ast.6 scanner.6
typechecker.6: ast.6 token.6
scanner.6: utils.6
scanner.6: token.6 utils.6
ast.6: scanner.6 symboltable.6
ast.6: token.6 symboltable.6
symboltable.6:
parser.6: scanner.6 ast.6 symboltable.6
parser.6: token.6 scanner.6 ast.6 symboltable.6
platform.6: utils.6
printer.6: utils.6 scanner.6 ast.6 symboltable.6
printer.6: utils.6 token.6 ast.6 symboltable.6
%.6: %.go
$(G) $(F) $<

12
usr/gri/pretty/ast.go
View File

@ -6,7 +6,7 @@ package AST
import (
"vector";
Scanner "scanner";
"token";
SymbolTable "symboltable";
)
@ -259,7 +259,7 @@ func ExprLen(x Expr) int {
}
n := 1;
for {
if p, ok := x.(*BinaryExpr); ok && p.Tok == Scanner.COMMA {
if p, ok := x.(*BinaryExpr); ok && p.Tok == token.COMMA {
n++;
x = p.Y;
} else {
@ -272,10 +272,10 @@ func ExprLen(x Expr) int {
func ExprAt(x Expr, i int) Expr {
for j := 0; j < i; j++ {
assert(x.(*BinaryExpr).Tok == Scanner.COMMA);
assert(x.(*BinaryExpr).Tok == token.COMMA);
x = x.(*BinaryExpr).Y;
}
if t, is_binary := x.(*BinaryExpr); is_binary && t.Tok == Scanner.COMMA {
if t, is_binary := x.(*BinaryExpr); is_binary && t.Tok == token.COMMA {
x = t.X;
}
return x;
@ -298,7 +298,7 @@ type Block struct {
func NewBlock(pos, tok int) *Block {
assert(tok == Scanner.LBRACE || tok == Scanner.COLON);
assert(tok == token.LBRACE || tok == token.COLON);
b := new(Block);
b.Pos, b.Tok, b.List = pos, tok, vector.New(0);
return b;
@ -509,7 +509,7 @@ func NewComment(pos int, text string) *Comment {
type Program struct {
Pos int; // tok is Scanner.PACKAGE
Pos int; // tok is token.PACKAGE
Ident Expr;
Decls []Decl;
Comments *vector.Vector;

408
usr/gri/pretty/parser.go
View File

File diff suppressed because it is too large Load Diff

162
usr/gri/pretty/printer.go
View File

@ -12,7 +12,7 @@ import (
"flag";
"fmt";
Utils "utils";
Scanner "scanner";
"token";
AST "ast";
SymbolTable "symboltable";
)
@ -136,7 +136,7 @@ func (P *Printer) Init(text io.Write, html bool, comments *vector.Vector) {
// formatting parameters & semantic state initialized correctly by default
// expression precedence
P.prec = Scanner.LowestPrec;
P.prec = token.LowestPrec;
}
@ -374,13 +374,13 @@ func (P *Printer) String(pos int, s string) {
func (P *Printer) Token(pos int, tok int) {
P.String(pos, Scanner.TokenString(tok));
//P.TaggedString(pos, "<b>", Scanner.TokenString(tok), "</b>");
P.String(pos, token.TokenString(tok));
//P.TaggedString(pos, "<b>", token.TokenString(tok), "</b>");
}
func (P *Printer) Error(pos int, tok int, msg string) {
fmt.Printf("\ninternal printing error: pos = %d, tok = %s, %s\n", pos, Scanner.TokenString(tok), msg);
fmt.Printf("\ninternal printing error: pos = %d, tok = %s, %s\n", pos, token.TokenString(tok), msg);
panic();
}
@ -456,7 +456,7 @@ func (P *Printer) Expr(x AST.Expr)
func (P *Printer) Idents(list []*AST.Ident) {
for i, x := range list {
if i > 0 {
P.Token(0, Scanner.COMMA);
P.Token(0, token.COMMA);
P.separator = blank;
P.state = inside_list;
}
@ -466,7 +466,7 @@ func (P *Printer) Idents(list []*AST.Ident) {
func (P *Printer) Parameters(list []*AST.Field) {
P.Token(0, Scanner.LPAREN);
P.Token(0, token.LPAREN);
if len(list) > 0 {
for i, par := range list {
if i > 0 {
@ -479,7 +479,7 @@ func (P *Printer) Parameters(list []*AST.Field) {
P.Expr(par.Typ);
}
}
P.Token(0, Scanner.RPAREN);
P.Token(0, token.RPAREN);
}
@ -508,7 +508,7 @@ func (P *Printer) Signature(sig *AST.Signature) {
func (P *Printer) Fields(list []*AST.Field, end int, is_interface bool) {
P.state = opening_scope;
P.separator = blank;
P.Token(0, Scanner.LBRACE);
P.Token(0, token.LBRACE);
if len(list) > 0 {
P.newlines = 1;
@ -539,7 +539,7 @@ func (P *Printer) Fields(list []*AST.Field, end int, is_interface bool) {
}
P.state = closing_scope;
P.Token(end, Scanner.RBRACE);
P.Token(end, token.RBRACE);
P.opt_semi = true;
}
@ -562,17 +562,17 @@ func (P *Printer) DoIdent(x *AST.Ident) {
func (P *Printer) DoBinaryExpr(x *AST.BinaryExpr) {
if x.Tok == Scanner.COMMA {
if x.Tok == token.COMMA {
// (don't use binary expression printing because of different spacing)
P.Expr(x.X);
P.Token(x.Pos_, Scanner.COMMA);
P.Token(x.Pos_, token.COMMA);
P.separator = blank;
P.state = inside_list;
P.Expr(x.Y);
} else {
prec := Scanner.Precedence(x.Tok);
prec := token.Precedence(x.Tok);
if prec < P.prec {
P.Token(0, Scanner.LPAREN);
P.Token(0, token.LPAREN);
}
P.Expr1(x.X, prec);
P.separator = blank;
@ -580,24 +580,24 @@ func (P *Printer) DoBinaryExpr(x *AST.BinaryExpr) {
P.separator = blank;
P.Expr1(x.Y, prec);
if prec < P.prec {
P.Token(0, Scanner.RPAREN);
P.Token(0, token.RPAREN);
}
}
}
func (P *Printer) DoUnaryExpr(x *AST.UnaryExpr) {
prec := Scanner.UnaryPrec;
prec := token.UnaryPrec;
if prec < P.prec {
P.Token(0, Scanner.LPAREN);
P.Token(0, token.LPAREN);
}
P.Token(x.Pos_, x.Tok);
if x.Tok == Scanner.RANGE {
if x.Tok == token.RANGE {
P.separator = blank;
}
P.Expr1(x.X, prec);
if prec < P.prec {
P.Token(0, Scanner.RPAREN);
P.Token(0, token.RPAREN);
}
}
@ -608,7 +608,7 @@ func (P *Printer) DoBasicLit(x *AST.BasicLit) {
func (P *Printer) DoFunctionLit(x *AST.FunctionLit) {
P.Token(x.Pos_, Scanner.FUNC);
P.Token(x.Pos_, token.FUNC);
P.Signature(x.Typ);
P.separator = blank;
P.Block(x.Body, true);
@ -617,64 +617,64 @@ func (P *Printer) DoFunctionLit(x *AST.FunctionLit) {
func (P *Printer) DoGroup(x *AST.Group) {
P.Token(x.Pos_, Scanner.LPAREN);
P.Token(x.Pos_, token.LPAREN);
P.Expr(x.X);
P.Token(0, Scanner.RPAREN);
P.Token(0, token.RPAREN);
}
func (P *Printer) DoSelector(x *AST.Selector) {
P.Expr1(x.X, Scanner.HighestPrec);
P.Token(x.Pos_, Scanner.PERIOD);
P.Expr1(x.Sel, Scanner.HighestPrec);
P.Expr1(x.X, token.HighestPrec);
P.Token(x.Pos_, token.PERIOD);
P.Expr1(x.Sel, token.HighestPrec);
}
func (P *Printer) DoTypeGuard(x *AST.TypeGuard) {
P.Expr1(x.X, Scanner.HighestPrec);
P.Token(x.Pos_, Scanner.PERIOD);
P.Token(0, Scanner.LPAREN);
P.Expr1(x.X, token.HighestPrec);
P.Token(x.Pos_, token.PERIOD);
P.Token(0, token.LPAREN);
P.Expr(x.Typ);
P.Token(0, Scanner.RPAREN);
P.Token(0, token.RPAREN);
}
func (P *Printer) DoIndex(x *AST.Index) {
P.Expr1(x.X, Scanner.HighestPrec);
P.Token(x.Pos_, Scanner.LBRACK);
P.Expr1(x.X, token.HighestPrec);
P.Token(x.Pos_, token.LBRACK);
P.Expr1(x.I, 0);
P.Token(0, Scanner.RBRACK);
P.Token(0, token.RBRACK);
}
func (P *Printer) DoCall(x *AST.Call) {
P.Expr1(x.F, Scanner.HighestPrec);
P.Expr1(x.F, token.HighestPrec);
P.Token(x.Pos_, x.Tok);
P.Expr(x.Args);
switch x.Tok {
case Scanner.LPAREN: P.Token(0, Scanner.RPAREN);
case Scanner.LBRACE: P.Token(0, Scanner.RBRACE);
case token.LPAREN: P.Token(0, token.RPAREN);
case token.LBRACE: P.Token(0, token.RBRACE);
}
}
func (P *Printer) DoEllipsis(x *AST.Ellipsis) {
P.Token(x.Pos_, Scanner.ELLIPSIS);
P.Token(x.Pos_, token.ELLIPSIS);
}
func (P *Printer) DoArrayType(x *AST.ArrayType) {
P.Token(x.Pos_, Scanner.LBRACK);
P.Token(x.Pos_, token.LBRACK);
if x.Len != nil {
P.Expr(x.Len);
}
P.Token(0, Scanner.RBRACK);
P.Token(0, token.RBRACK);
P.Expr(x.Elt);
}
func (P *Printer) DoStructType(x *AST.StructType) {
P.Token(x.Pos_, Scanner.STRUCT);
P.Token(x.Pos_, token.STRUCT);
if x.End > 0 {
P.Fields(x.Fields, x.End, false);
}
@ -682,19 +682,19 @@ func (P *Printer) DoStructType(x *AST.StructType) {
func (P *Printer) DoPointerType(x *AST.PointerType) {
P.Token(x.Pos_, Scanner.MUL);
P.Token(x.Pos_, token.MUL);
P.Expr(x.Base);
}
func (P *Printer) DoFunctionType(x *AST.FunctionType) {
P.Token(x.Pos_, Scanner.FUNC);
P.Token(x.Pos_, token.FUNC);
P.Signature(x.Sig);
}
func (P *Printer) DoInterfaceType(x *AST.InterfaceType) {
P.Token(x.Pos_, Scanner.INTERFACE);
P.Token(x.Pos_, token.INTERFACE);
if x.End > 0 {
P.Fields(x.Methods, x.End, true);
}
@ -707,11 +707,11 @@ func (P *Printer) DoSliceType(x *AST.SliceType) {
func (P *Printer) DoMapType(x *AST.MapType) {
P.Token(x.Pos_, Scanner.MAP);
P.Token(x.Pos_, token.MAP);
P.separator = blank;
P.Token(0, Scanner.LBRACK);
P.Token(0, token.LBRACK);
P.Expr(x.Key);
P.Token(0, Scanner.RBRACK);
P.Token(0, token.RBRACK);
P.Expr(x.Val);
}
@ -719,14 +719,14 @@ func (P *Printer) DoMapType(x *AST.MapType) {
func (P *Printer) DoChannelType(x *AST.ChannelType) {
switch x.Mode {
case AST.FULL:
P.Token(x.Pos_, Scanner.CHAN);
P.Token(x.Pos_, token.CHAN);
case AST.RECV:
P.Token(x.Pos_, Scanner.ARROW);
P.Token(0, Scanner.CHAN);
P.Token(x.Pos_, token.ARROW);
P.Token(0, token.CHAN);
case AST.SEND:
P.Token(x.Pos_, Scanner.CHAN);
P.Token(x.Pos_, token.CHAN);
P.separator = blank;
P.Token(0, Scanner.ARROW);
P.Token(0, token.ARROW);
}
P.separator = blank;
P.Expr(x.Val);
@ -746,7 +746,7 @@ func (P *Printer) Expr1(x AST.Expr, prec1 int) {
func (P *Printer) Expr(x AST.Expr) {
P.Expr1(x, Scanner.LowestPrec);
P.Expr1(x, token.LowestPrec);
}
@ -789,8 +789,8 @@ func (P *Printer) Block(b *AST.Block, indent bool) {
P.separator = none;
}
P.state = closing_scope;
if b.Tok == Scanner.LBRACE {
P.Token(b.End, Scanner.RBRACE);
if b.Tok == token.LBRACE {
P.Token(b.End, token.RBRACE);
P.opt_semi = true;
} else {
P.String(0, ""); // process closing_scope state transition!
@ -808,7 +808,7 @@ func (P *Printer) DoBadStat(s *AST.BadStat) {
func (P *Printer) DoLabelDecl(s *AST.LabelDecl) {
P.indentation--;
P.Expr(s.Label);
P.Token(s.Pos, Scanner.COLON);
P.Token(s.Pos, token.COLON);
// TODO not quite correct:
// - we must not print this optional semicolon, as it may invalidate code.
// - this will change once the AST reflects the LabelStatement change
@ -824,12 +824,12 @@ func (P *Printer) DoDeclarationStat(s *AST.DeclarationStat) {
func (P *Printer) DoExpressionStat(s *AST.ExpressionStat) {
switch s.Tok {
case Scanner.ILLEGAL:
case token.ILLEGAL:
P.Expr(s.Expr);
case Scanner.INC, Scanner.DEC:
case token.INC, token.DEC:
P.Expr(s.Expr);
P.Token(s.Pos, s.Tok);
case Scanner.RETURN, Scanner.GO, Scanner.DEFER:
case token.RETURN, token.GO, token.DEFER:
P.Token(s.Pos, s.Tok);
if s.Expr != nil {
P.separator = blank;
@ -861,14 +861,14 @@ func (P *Printer) ControlClause(isForStat bool, init AST.Stat, expr AST.Expr, po
P.Stat(init);
P.separator = none;
}
P.Token(0, Scanner.SEMICOLON);
P.Token(0, token.SEMICOLON);
P.separator = blank;
if expr != nil {
P.Expr(expr);
P.separator = none;
}
if isForStat {
P.Token(0, Scanner.SEMICOLON);
P.Token(0, token.SEMICOLON);
P.separator = blank;
if post != nil {
P.Stat(post);
@ -880,12 +880,12 @@ func (P *Printer) ControlClause(isForStat bool, init AST.Stat, expr AST.Expr, po
func (P *Printer) DoIfStat(s *AST.IfStat) {
P.Token(s.Pos, Scanner.IF);
P.Token(s.Pos, token.IF);
P.ControlClause(false, s.Init, s.Cond, nil);
P.Block(s.Body, true);
if s.Else != nil {
P.separator = blank;
P.Token(0, Scanner.ELSE);
P.Token(0, token.ELSE);
P.separator = blank;
P.Stat(s.Else);
}
@ -893,7 +893,7 @@ func (P *Printer) DoIfStat(s *AST.IfStat) {
func (P *Printer) DoForStat(s *AST.ForStat) {
P.Token(s.Pos, Scanner.FOR);
P.Token(s.Pos, token.FOR);
P.ControlClause(true, s.Init, s.Cond, s.Post);
P.Block(s.Body, true);
}
@ -901,15 +901,15 @@ func (P *Printer) DoForStat(s *AST.ForStat) {
func (P *Printer) DoCaseClause(s *AST.CaseClause) {
if s.Expr != nil {
P.Token(s.Pos, Scanner.CASE);
P.Token(s.Pos, token.CASE);
P.separator = blank;
P.Expr(s.Expr);
} else {
P.Token(s.Pos, Scanner.DEFAULT);
P.Token(s.Pos, token.DEFAULT);
}
// TODO: try to use P.Block instead
// P.Block(s.Body, true);
P.Token(s.Body.Pos, Scanner.COLON);
P.Token(s.Body.Pos, token.COLON);
P.indentation++;
P.StatementList(s.Body.List);
P.indentation--;
@ -918,14 +918,14 @@ func (P *Printer) DoCaseClause(s *AST.CaseClause) {
func (P *Printer) DoSwitchStat(s *AST.SwitchStat) {
P.Token(s.Pos, Scanner.SWITCH);
P.Token(s.Pos, token.SWITCH);
P.ControlClause(false, s.Init, s.Tag, nil);
P.Block(s.Body, false);
}
func (P *Printer) DoSelectStat(s *AST.SelectStat) {
P.Token(s.Pos, Scanner.SELECT);
P.Token(s.Pos, token.SELECT);
P.separator = blank;
P.Block(s.Body, false);
}
@ -955,7 +955,7 @@ func (P *Printer) DoBadDecl(d *AST.BadDecl) {
func (P *Printer) DoImportDecl(d *AST.ImportDecl) {
if d.Pos > 0 {
P.Token(d.Pos, Scanner.IMPORT);
P.Token(d.Pos, token.IMPORT);
P.separator = blank;
}
if d.Ident != nil {
@ -964,7 +964,7 @@ func (P *Printer) DoImportDecl(d *AST.ImportDecl) {
P.String(d.Path.Pos(), ""); // flush pending ';' separator/newlines
}
P.separator = tab;
if lit, is_lit := d.Path.(*AST.BasicLit); is_lit && lit.Tok == Scanner.STRING {
if lit, is_lit := d.Path.(*AST.BasicLit); is_lit && lit.Tok == token.STRING {
P.HtmlPackageName(lit.Pos_, lit.Val);
} else {
// we should only reach here for strange imports
@ -977,7 +977,7 @@ func (P *Printer) DoImportDecl(d *AST.ImportDecl) {
func (P *Printer) DoConstDecl(d *AST.ConstDecl) {
if d.Pos > 0 {
P.Token(d.Pos, Scanner.CONST);
P.Token(d.Pos, token.CONST);
P.separator = blank;
}
P.Idents(d.Idents);
@ -987,7 +987,7 @@ func (P *Printer) DoConstDecl(d *AST.ConstDecl) {
}
if d.Vals != nil {
P.separator = tab;
P.Token(0, Scanner.ASSIGN);
P.Token(0, token.ASSIGN);
P.separator = blank;
P.Expr(d.Vals);
}
@ -997,7 +997,7 @@ func (P *Printer) DoConstDecl(d *AST.ConstDecl) {
func (P *Printer) DoTypeDecl(d *AST.TypeDecl) {
if d.Pos > 0 {
P.Token(d.Pos, Scanner.TYPE);
P.Token(d.Pos, token.TYPE);
P.separator = blank;
}
P.Expr(d.Ident);
@ -1009,7 +1009,7 @@ func (P *Printer) DoTypeDecl(d *AST.TypeDecl) {
func (P *Printer) DoVarDecl(d *AST.VarDecl) {
if d.Pos > 0 {
P.Token(d.Pos, Scanner.VAR);
P.Token(d.Pos, token.VAR);
P.separator = blank;
}
P.Idents(d.Idents);
@ -1020,7 +1020,7 @@ func (P *Printer) DoVarDecl(d *AST.VarDecl) {
}
if d.Vals != nil {
P.separator = tab;
P.Token(0, Scanner.ASSIGN);
P.Token(0, token.ASSIGN);
P.separator = blank;
P.Expr(d.Vals);
}
@ -1029,17 +1029,17 @@ func (P *Printer) DoVarDecl(d *AST.VarDecl) {
func (P *Printer) DoFuncDecl(d *AST.FuncDecl) {
P.Token(d.Pos_, Scanner.FUNC);
P.Token(d.Pos_, token.FUNC);
P.separator = blank;
if recv := d.Recv; recv != nil {
// method: print receiver
P.Token(0, Scanner.LPAREN);
P.Token(0, token.LPAREN);
if len(recv.Idents) > 0 {
P.Expr(recv.Idents[0]);
P.separator = blank;
}
P.Expr(recv.Typ);
P.Token(0, Scanner.RPAREN);
P.Token(0, token.RPAREN);
P.separator = blank;
}
P.Expr(d.Ident);
@ -1053,7 +1053,7 @@ func (P *Printer) DoFuncDecl(d *AST.FuncDecl) {
func (P *Printer) DoDeclList(d *AST.DeclList) {
if !*def || d.Tok == Scanner.IMPORT || d.Tok == Scanner.VAR {
if !*def || d.Tok == token.IMPORT || d.Tok == token.VAR {
P.Token(d.Pos, d.Tok);
} else {
P.String(d.Pos, "def");
@ -1062,7 +1062,7 @@ func (P *Printer) DoDeclList(d *AST.DeclList) {
// group of parenthesized declarations
P.state = opening_scope;
P.Token(0, Scanner.LPAREN);
P.Token(0, token.LPAREN);
if len(d.List) > 0 {
P.newlines = 1;
for i := 0; i < len(d.List); i++ {
@ -1074,7 +1074,7 @@ func (P *Printer) DoDeclList(d *AST.DeclList) {
}
}
P.state = closing_scope;
P.Token(d.End, Scanner.RPAREN);
P.Token(d.End, token.RPAREN);
P.opt_semi = true;
P.newlines = 2;
}
@ -1089,7 +1089,7 @@ func (P *Printer) Decl(d AST.Decl) {
// Program
func (P *Printer) Program(p *AST.Program) {
P.Token(p.Pos, Scanner.PACKAGE);
P.Token(p.Pos, token.PACKAGE);
P.separator = blank;
P.Expr(p.Ident);
P.newlines = 1;

420
usr/gri/pretty/scanner.go
View File

@ -4,272 +4,33 @@
package scanner
// A Go scanner. Takes a []byte as source which can then be
// tokenized through repeated calls to the Scan() function.
//
// Sample use:
//
// import "token"
// import "scanner"
//
// func tokenize(src []byte) {
// var s scanner.Scanner;
// s.Init(src, nil, false);
// for {
// pos, tok, lit := s.Scan();
// if tok == Scanner.EOF {
// return;
// }
// println(pos, token.TokenString(tok), string(lit));
// }
// }
import (
"utf8";
"unicode";
"strconv";
"token";
)
const (
ILLEGAL = iota;
EOF;
INT;
FLOAT;
STRING;
IDENT;
COMMENT;
ADD;
SUB;
MUL;
QUO;
REM;
AND;
OR;
XOR;
SHL;
SHR;
ADD_ASSIGN;
SUB_ASSIGN;
MUL_ASSIGN;
QUO_ASSIGN;
REM_ASSIGN;
AND_ASSIGN;
OR_ASSIGN;
XOR_ASSIGN;
SHL_ASSIGN;
SHR_ASSIGN;
LAND;
LOR;
ARROW;
INC;
DEC;
EQL;
LSS;
GTR;
ASSIGN;
NOT;
NEQ;
LEQ;
GEQ;
DEFINE;
ELLIPSIS;
LPAREN;
LBRACK;
LBRACE;
COMMA;
PERIOD;
RPAREN;
RBRACK;
RBRACE;
SEMICOLON;
COLON;
// keywords
keywords_beg;
BREAK;
CASE;
CHAN;
CONST;
CONTINUE;
DEFAULT;
DEFER;
ELSE;
FALLTHROUGH;
FOR;
FUNC;
GO;
GOTO;
IF;
IMPORT;
INTERFACE;
MAP;
PACKAGE;
RANGE;
RETURN;
SELECT;
STRUCT;
SWITCH;
TYPE;
VAR;
keywords_end;
)
func TokenString(tok int) string {
switch tok {
case ILLEGAL: return "ILLEGAL";
case EOF: return "EOF";
case INT: return "INT";
case FLOAT: return "FLOAT";
case STRING: return "STRING";
case IDENT: return "IDENT";
case COMMENT: return "COMMENT";
case ADD: return "+";
case SUB: return "-";
case MUL: return "*";
case QUO: return "/";
case REM: return "%";
case AND: return "&";
case OR: return "|";
case XOR: return "^";
case SHL: return "<<";
case SHR: return ">>";
case ADD_ASSIGN: return "+=";
case SUB_ASSIGN: return "-=";
case MUL_ASSIGN: return "+=";
case QUO_ASSIGN: return "/=";
case REM_ASSIGN: return "%=";
case AND_ASSIGN: return "&=";
case OR_ASSIGN: return "|=";
case XOR_ASSIGN: return "^=";
case SHL_ASSIGN: return "<<=";
case SHR_ASSIGN: return ">>=";
case LAND: return "&&";
case LOR: return "||";
case ARROW: return "<-";
case INC: return "++";
case DEC: return "--";
case EQL: return "==";
case LSS: return "<";
case GTR: return ">";
case ASSIGN: return "=";
case NOT: return "!";
case NEQ: return "!=";
case LEQ: return "<=";
case GEQ: return ">=";
case DEFINE: return ":=";
case ELLIPSIS: return "...";
case LPAREN: return "(";
case LBRACK: return "[";
case LBRACE: return "{";
case COMMA: return ",";
case PERIOD: return ".";
case RPAREN: return ")";
case RBRACK: return "]";
case RBRACE: return "}";
case SEMICOLON: return ";";
case COLON: return ":";
case BREAK: return "break";
case CASE: return "case";
case CHAN: return "chan";
case CONST: return "const";
case CONTINUE: return "continue";
case DEFAULT: return "default";
case DEFER: return "defer";
case ELSE: return "else";
case FALLTHROUGH: return "fallthrough";
case FOR: return "for";
case FUNC: return "func";
case GO: return "go";
case GOTO: return "goto";
case IF: return "if";
case IMPORT: return "import";
case INTERFACE: return "interface";
case MAP: return "map";
case PACKAGE: return "package";
case RANGE: return "range";
case RETURN: return "return";
case SELECT: return "select";
case STRUCT: return "struct";
case SWITCH: return "switch";
case TYPE: return "type";
case VAR: return "var";
}
return "token(" + strconv.Itoa(tok) + ")";
}
const (
LowestPrec = -1;
UnaryPrec = 7;
HighestPrec = 8;
)
func Precedence(tok int) int {
switch tok {
case COLON:
return 0;
case LOR:
return 1;
case LAND:
return 2;
case ARROW:
return 3;
case EQL, NEQ, LSS, LEQ, GTR, GEQ:
return 4;
case ADD, SUB, OR, XOR:
return 5;
case MUL, QUO, REM, SHL, SHR, AND:
return 6;
}
return LowestPrec;
}
var keywords map [string] int;
func init() {
keywords = make(map [string] int);
for i := keywords_beg + 1; i < keywords_end; i++ {
keywords[TokenString(i)] = i;
}
}
func is_letter(ch int) bool {
return
'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || // common case
ch == '_' || unicode.IsLetter(ch);
}
func digit_val(ch int) int {
// TODO: spec permits other Unicode digits as well
if '0' <= ch && ch <= '9' {
return ch - '0';
}
if 'a' <= ch && ch <= 'f' {
return ch - 'a' + 10;
}
if 'A' <= ch && ch <= 'F' {
return ch - 'A' + 10;
}
return 16; // larger than any legal digit val
}
type ErrorHandler interface {
Error(pos int, msg string);
}
@ -288,6 +49,28 @@ type Scanner struct {
}
func is_letter(ch int) bool {
return
'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || // common case
ch == '_' || unicode.IsLetter(ch);
}
func digit_val(ch int) int {
// TODO spec permits other Unicode digits as well
if '0' <= ch && ch <= '9' {
return ch - '0';
}
if 'a' <= ch && ch <= 'f' {
return ch - 'a' + 10;
}
if 'A' <= ch && ch <= 'F' {
return ch - 'A' + 10;
}
return 16; // larger than any legal digit val
}
// Read the next Unicode char into S.ch.
// S.ch < 0 means end-of-file.
func (S *Scanner) next() {
@ -308,10 +91,12 @@ func (S *Scanner) next() {
}
func (S *Scanner) error(pos int, msg string) {
S.err.Error(pos, msg);
}
// Initialize the scanner.
//
// The error handler (err) is called when an illegal token is encountered.
// If scan_comments is set to true, newline characters ('\n') and comments
// are recognized as token.COMMENT, otherwise they are treated as white
// space and ignored.
func (S *Scanner) Init(src []byte, err ErrorHandler, scan_comments bool) {
S.src = src;
@ -338,6 +123,11 @@ func charString(ch int) string {
}
func (S *Scanner) error(pos int, msg string) {
S.err.Error(pos, msg);
}
func (S *Scanner) expect(ch int) {
if S.ch != ch {
S.error(S.chpos, "expected " + charString(ch) + ", found " + charString(S.ch));
@ -400,20 +190,13 @@ exit:
}
func (S *Scanner) scanIdentifier() (tok int, val []byte) {
func (S *Scanner) scanIdentifier() (tok int, lit []byte) {
pos := S.chpos;
for is_letter(S.ch) || digit_val(S.ch) < 10 {
S.next();
}
val = S.src[pos : S.chpos];
var present bool;
tok, present = keywords[string(val)];
if !present {
tok = IDENT;
}
return tok, val;
lit = S.src[pos : S.chpos];
return token.Lookup(lit), lit;
}
@ -424,12 +207,12 @@ func (S *Scanner) scanMantissa(base int) {
}
func (S *Scanner) scanNumber(seen_decimal_point bool) (tok int, val []byte) {
func (S *Scanner) scanNumber(seen_decimal_point bool) (tok int, lit []byte) {
pos := S.chpos;
tok = INT;
tok = token.INT;
if seen_decimal_point {
tok = FLOAT;
tok = token.FLOAT;
pos--; // '.' is one byte
S.scanMantissa(10);
goto exponent;
@ -447,7 +230,7 @@ func (S *Scanner) scanNumber(seen_decimal_point bool) (tok int, val []byte) {
S.scanMantissa(8);
if digit_val(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
// float
tok = FLOAT;
tok = token.FLOAT;
goto mantissa;
}
// octal int
@ -461,7 +244,7 @@ mantissa:
if S.ch == '.' {
// float
tok = FLOAT;
tok = token.FLOAT;
S.next();
S.scanMantissa(10)
}
@ -469,7 +252,7 @@ mantissa:
exponent:
if S.ch == 'e' || S.ch == 'E' {
// float
tok = FLOAT;
tok = token.FLOAT;
S.next();
if S.ch == '-' || S.ch == '+' {
S.next();
@ -607,75 +390,78 @@ func (S *Scanner) select4(tok0, tok1, ch2, tok2, tok3 int) int {
}
func (S *Scanner) Scan() (pos, tok int, val []byte) {
// Scans the next token. Returns the token byte position in the source,
// its token value, and the corresponding literal text if the token is
// an identifier or basic type literals (token.IsLiteral(tok) == true).
func (S *Scanner) Scan() (pos, tok int, lit []byte) {
loop:
S.skipWhitespace();
pos, tok = S.chpos, ILLEGAL;
pos, tok = S.chpos, token.ILLEGAL;
switch ch := S.ch; {
case is_letter(ch): tok, val = S.scanIdentifier();
case digit_val(ch) < 10: tok, val = S.scanNumber(false);
case is_letter(ch): tok, lit = S.scanIdentifier();
case digit_val(ch) < 10: tok, lit = S.scanNumber(false);
default:
S.next(); // always make progress
switch ch {
case -1: tok = EOF;
case '\n': tok, val = COMMENT, []byte{'\n'};
case '"': tok, val = STRING, S.scanString();
case '\'': tok, val = INT, S.scanChar();
case '`': tok, val = STRING, S.scanRawString();
case ':': tok = S.select2(COLON, DEFINE);
case -1: tok = token.EOF;
case '\n': tok, lit = token.COMMENT, []byte{'\n'};
case '"': tok, lit = token.STRING, S.scanString();
case '\'': tok, lit = token.CHAR, S.scanChar();
case '`': tok, lit = token.STRING, S.scanRawString();
case ':': tok = S.select2(token.COLON, token.DEFINE);
case '.':
if digit_val(S.ch) < 10 {
tok, val = S.scanNumber(true);
tok, lit = S.scanNumber(true);
} else if S.ch == '.' {
S.next();
if S.ch == '.' {
S.next();
tok = ELLIPSIS;
tok = token.ELLIPSIS;
}
} else {
tok = PERIOD;
tok = token.PERIOD;
}
case ',': tok = COMMA;
case ';': tok = SEMICOLON;
case '(': tok = LPAREN;
case ')': tok = RPAREN;
case '[': tok = LBRACK;
case ']': tok = RBRACK;
case '{': tok = LBRACE;
case '}': tok = RBRACE;
case '+': tok = S.select3(ADD, ADD_ASSIGN, '+', INC);
case '-': tok = S.select3(SUB, SUB_ASSIGN, '-', DEC);
case '*': tok = S.select2(MUL, MUL_ASSIGN);
case ',': tok = token.COMMA;
case ';': tok = token.SEMICOLON;
case '(': tok = token.LPAREN;
case ')': tok = token.RPAREN;
case '[': tok = token.LBRACK;
case ']': tok = token.RBRACK;
case '{': tok = token.LBRACE;
case '}': tok = token.RBRACE;
case '+': tok = S.select3(token.ADD, token.ADD_ASSIGN, '+', token.INC);
case '-': tok = S.select3(token.SUB, token.SUB_ASSIGN, '-', token.DEC);
case '*': tok = S.select2(token.MUL, token.MUL_ASSIGN);
case '/':
if S.ch == '/' || S.ch == '*' {
tok, val = COMMENT, S.scanComment();
tok, lit = token.COMMENT, S.scanComment();
if !S.scan_comments {
goto loop;
}
} else {
tok = S.select2(QUO, QUO_ASSIGN);
tok = S.select2(token.QUO, token.QUO_ASSIGN);
}
case '%': tok = S.select2(REM, REM_ASSIGN);
case '^': tok = S.select2(XOR, XOR_ASSIGN);
case '%': tok = S.select2(token.REM, token.REM_ASSIGN);
case '^': tok = S.select2(token.XOR, token.XOR_ASSIGN);
case '<':
if S.ch == '-' {
S.next();
tok = ARROW;
tok = token.ARROW;
} else {
tok = S.select4(LSS, LEQ, '<', SHL, SHL_ASSIGN);
tok = S.select4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN);
}
case '>': tok = S.select4(GTR, GEQ, '>', SHR, SHR_ASSIGN);
case '=': tok = S.select2(ASSIGN, EQL);
case '!': tok = S.select2(NOT, NEQ);
case '&': tok = S.select3(AND, AND_ASSIGN, '&', LAND);
case '|': tok = S.select3(OR, OR_ASSIGN, '|', LOR);
case '>': tok = S.select4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN);
case '=': tok = S.select2(token.ASSIGN, token.EQL);
case '!': tok = S.select2(token.NOT, token.NEQ);
case '&': tok = S.select3(token.AND, token.AND_ASSIGN, '&', token.LAND);
case '|': tok = S.select3(token.OR, token.OR_ASSIGN, '|', token.LOR);
default:
S.error(pos, "illegal character " + charString(ch));
tok = ILLEGAL;
}
}
return pos, tok, val;
return pos, tok, lit;
}

286
usr/gri/pretty/token.go Normal file
View File

@ -0,0 +1,286 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package token
// Defines Go tokens and basic token operations.
import "strconv";
const (
// Special tokens
ILLEGAL = iota;
EOF;
COMMENT;
// Identifiers and basic type literals
// (these tokens stand for classes of literals)
literal_beg;
IDENT;
INT;
FLOAT;
CHAR;
STRING;
literal_end;
// Operators and delimiters
operator_beg;
ADD;
SUB;
MUL;
QUO;
REM;
AND;
OR;
XOR;
SHL;
SHR;
ADD_ASSIGN;
SUB_ASSIGN;
MUL_ASSIGN;
QUO_ASSIGN;
REM_ASSIGN;
AND_ASSIGN;
OR_ASSIGN;
XOR_ASSIGN;
SHL_ASSIGN;
SHR_ASSIGN;
LAND;
LOR;
ARROW;
INC;
DEC;
EQL;
LSS;
GTR;
ASSIGN;
NOT;
NEQ;
LEQ;
GEQ;
DEFINE;
ELLIPSIS;
LPAREN;
LBRACK;
LBRACE;
COMMA;
PERIOD;
RPAREN;
RBRACK;
RBRACE;
SEMICOLON;
COLON;
operator_end;
// Keywords
keyword_beg;
BREAK;
CASE;
CHAN;
CONST;
CONTINUE;
DEFAULT;
DEFER;
ELSE;
FALLTHROUGH;
FOR;
FUNC;
GO;
GOTO;
IF;
IMPORT;
INTERFACE;
MAP;
PACKAGE;
RANGE;
RETURN;
SELECT;
STRUCT;
SWITCH;
TYPE;
VAR;
keyword_end;
)
func TokenString(tok int) string {
switch tok {
case ILLEGAL: return "ILLEGAL";
case EOF: return "EOF";
case COMMENT: return "COMMENT";
case IDENT: return "IDENT";
case INT: return "INT";
case FLOAT: return "FLOAT";
case CHAR: return "CHAR";
case STRING: return "STRING";
case ADD: return "+";
case SUB: return "-";
case MUL: return "*";
case QUO: return "/";
case REM: return "%";
case AND: return "&";
case OR: return "|";
case XOR: return "^";
case SHL: return "<<";
case SHR: return ">>";
case ADD_ASSIGN: return "+=";
case SUB_ASSIGN: return "-=";
case MUL_ASSIGN: return "+=";
case QUO_ASSIGN: return "/=";
case REM_ASSIGN: return "%=";
case AND_ASSIGN: return "&=";
case OR_ASSIGN: return "|=";
case XOR_ASSIGN: return "^=";
case SHL_ASSIGN: return "<<=";
case SHR_ASSIGN: return ">>=";
case LAND: return "&&";
case LOR: return "||";
case ARROW: return "<-";
case INC: return "++";
case DEC: return "--";
case EQL: return "==";
case LSS: return "<";
case GTR: return ">";
case ASSIGN: return "=";
case NOT: return "!";
case NEQ: return "!=";
case LEQ: return "<=";
case GEQ: return ">=";
case DEFINE: return ":=";
case ELLIPSIS: return "...";
case LPAREN: return "(";
case LBRACK: return "[";
case LBRACE: return "{";
case COMMA: return ",";
case PERIOD: return ".";
case RPAREN: return ")";
case RBRACK: return "]";
case RBRACE: return "}";
case SEMICOLON: return ";";
case COLON: return ":";
case BREAK: return "break";
case CASE: return "case";
case CHAN: return "chan";
case CONST: return "const";
case CONTINUE: return "continue";
case DEFAULT: return "default";
case DEFER: return "defer";
case ELSE: return "else";
case FALLTHROUGH: return "fallthrough";
case FOR: return "for";
case FUNC: return "func";
case GO: return "go";
case GOTO: return "goto";
case IF: return "if";
case IMPORT: return "import";
case INTERFACE: return "interface";
case MAP: return "map";
case PACKAGE: return "package";
case RANGE: return "range";
case RETURN: return "return";
case SELECT: return "select";
case STRUCT: return "struct";
case SWITCH: return "switch";
case TYPE: return "type";
case VAR: return "var";
}
return "token(" + strconv.Itoa(tok) + ")";
}
const (
LowestPrec = -1;
UnaryPrec = 7;
HighestPrec = 8;
)
func Precedence(tok int) int {
switch tok {
case COLON:
return 0;
case LOR:
return 1;
case LAND:
return 2;
case ARROW:
return 3;
case EQL, NEQ, LSS, LEQ, GTR, GEQ:
return 4;
case ADD, SUB, OR, XOR:
return 5;
case MUL, QUO, REM, SHL, SHR, AND:
return 6;
}
return LowestPrec;
}
var keywords map [string] int;
func init() {
keywords = make(map [string] int);
for i := keyword_beg + 1; i < keyword_end; i++ {
keywords[TokenString(i)] = i;
}
}
// Map an identifier to its keyword token or IDENT (if not a keyword).
func Lookup(ident []byte) int {
// TODO should not have to convert every ident into a string
// for lookup - but at the moment maps of []byte don't
// seem to work - gri 3/3/09
if tok, is_keyword := keywords[string(ident)]; is_keyword {
return tok;
}
return IDENT;
}
// Predicates
// Identifiers and basic type literals
func IsLiteral(tok int) bool {
return literal_beg < tok && tok < literal_end;
}
// Operators and delimiters
func IsOperator(tok int) bool {
return operator_beg < tok && tok < operator_end;
}
func IsKeyword(tok int) bool {
return keyword_beg < tok && tok < keyword_end;
}

25
usr/gri/pretty/typechecker.go
View File

@ -5,18 +5,23 @@
package TypeChecker
import (
"token";
AST "ast";
Scanner "scanner";
)
type ErrorHandler interface {
Error(pos int, msg string);
}
type state struct {
// setup
err Scanner.ErrorHandler;
err ErrorHandler;
}
func (s *state) Init(err Scanner.ErrorHandler) {
func (s *state) Init(err ErrorHandler) {
s.err = err;
}
@ -54,7 +59,7 @@ func (s *state) CheckType() {
/*
func (s *state) CheckDeclaration(d *AST.Decl) {
if d.Tok != Scanner.FUNC && d.List != nil {
if d.Tok != token.FUNC && d.List != nil {
// group of parenthesized declarations
for i := 0; i < d.List.Len(); i++ {
s.CheckDeclaration(d.List.At(i).(*AST.Decl))
@ -63,11 +68,11 @@ func (s *state) CheckDeclaration(d *AST.Decl) {
} else {
// single declaration
switch d.Tok {
case Scanner.IMPORT:
case Scanner.CONST:
case Scanner.VAR:
case Scanner.TYPE:
case Scanner.FUNC:
case token.IMPORT:
case token.CONST:
case token.VAR:
case token.TYPE:
case token.FUNC:
default:
unreachable();
}
@ -85,7 +90,7 @@ func (s *state) CheckProgram(p *AST.Program) {
// ----------------------------------------------------------------------------
func CheckProgram(err Scanner.ErrorHandler, p *AST.Program) {
func CheckProgram(err ErrorHandler, p *AST.Program) {
var s state;
s.Init(err);
s.CheckProgram(p);