§1.3 final passing version.

lexer/lexer.go

@@ -7,7 +7,7 @@ package lexer
 
 import (
 	"fgm/waiig15/token"
-	)
+)
 
 type Lexer struct {
 	input        string
@@ -36,6 +36,8 @@ func (l *Lexer) readChar() {
 func (l *Lexer) NextToken() token.Token {
 	var tok token.Token
 
+	l.skipWhitespace()
+
 	switch l.ch {
 	case '=':
 		tok = newToken(token.ASSIGN, l.ch)
@@ -56,6 +58,20 @@ func (l *Lexer) NextToken() token.Token {
 	case 0:
 		tok.Literal = ""
 		tok.Type = token.EOF
+	default:
+		if isLetter(l.ch) {
+			tok.Literal = l.readIdentifier()
+			tok.Type = token.LookupIdent(tok.Literal)
+			// We already read the next char, so avoid the final readChar().
+			return tok
+		} else if isDigit(l.ch) {
+			tok.Type = token.INT
+			tok.Literal = l.readNumber()
+			// Ditto.
+			return tok
+		} else {
+			tok = newToken(token.ILLEGAL, l.ch)
+		}
 	}
 
 	l.readChar()
@@ -63,5 +79,35 @@ func (l *Lexer) NextToken() token.Token {
 }
 
 func newToken(tokenType token.TokenType, ch byte) token.Token {
-	return token.Token{ Type: tokenType, Literal: string(ch) }
+	return token.Token{Type: tokenType, Literal: string(ch)}
+}
+
+func (l *Lexer) readIdentifier() string {
+	position := l.position
+	for isLetter(l.ch) {
+		l.readChar()
+	}
+	return l.input[position:l.position]
+}
+
+func (l *Lexer) readNumber() string {
+	position := l.position
+	for isDigit(l.ch) {
+		l.readChar()
+	}
+	return l.input[position:l.position]
+}
+
+func isDigit(ch byte) bool {
+	return '0' <= ch && ch <= '9'
+}
+
+func isLetter(ch byte) bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
+}
+
+func (l *Lexer) skipWhitespace() {
+	for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
+		l.readChar()
+	}
 }

lexer/lexer_test.go

@@ -6,19 +6,58 @@ import (
 )
 
 func TestNextToken(t *testing.T) {
-	input := `=+(){},;`
+	input := `
+let five = 5;
+let ten  = 10;
+
+let add = fn(x, y) {
+  x + y;
+}
+
+let result = add(five, ten);
+`
 
 	tests := []struct {
 		expectedType    token.TokenType
 		expectedLiteral string
 	}{
-		{token.ASSIGN, "="},
-		{token.PLUS, "+"},
-		{token.LPAREN, "("},
-		{token.RPAREN, ")"},
-		{token.LBRACE, "{"},
-		{token.RBRACE, "}"},
-		{token.COMMA, ","},
+		{ token.LET, "let" },
+		{ token.IDENT, "five" },
+		{ token.ASSIGN, "="},
+		{ token.INT, "5" },
+		{ token.SEMICOLON, ";" },
+
+		{ token.LET, "let" },
+		{ token.IDENT, "ten" },
+		{ token.ASSIGN, "="},
+		{ token.INT, "10" },
+		{ token.SEMICOLON, ";" },
+
+		{ token.LET, "let" },
+		{ token.IDENT, "add" },
+		{ token.ASSIGN, "="},
+		{ token.FUNCTION, "fn" },
+		{ token.LPAREN, "(" },
+		{ token.IDENT, "x" },
+		{ token.COMMA, "," },
+		{ token.IDENT, "y" },
+		{ token.RPAREN, ")" },
+		{ token.LBRACE, "{" },
+		{ token.IDENT, "x" },
+		{ token.PLUS, "+" },
+		{ token.IDENT, "y" },
+		{ token.SEMICOLON, ";" },
+		{ token.RBRACE, "}" },
+
+		{ token.LET, "let" },
+		{ token.IDENT, "result" },
+		{ token.ASSIGN, "="},
+		{ token.IDENT, "add" },
+		{ token.LPAREN, "(" },
+		{ token.IDENT, "five" },
+		{ token.COMMA, "," },
+		{ token.IDENT, "ten" },
+		{ token.RPAREN, ")" },
 		{token.SEMICOLON, ";"},
 		{token.EOF, ""},
 	}

token/token.go

@@ -3,7 +3,7 @@ package token
 type TokenType string
 
 type Token struct {
-	Type   TokenType
+	Type    TokenType
 	Literal string
 }
 
@@ -32,3 +32,17 @@ const (
 	FUNCTION = "FUNCTION"
 	LET      = "LET"
 )
+
+var keywords = map[string]TokenType{
+	"fn":  FUNCTION,
+	"let": LET,
+}
+
+// LookupIdent checks the keywords table to see whether the given identifier is
+// in fact a keyword.
+func LookupIdent(ident string) TokenType {
+	if tok, ok := keywords[ident]; ok {
+		return tok
+	}
+	return IDENT
+}