From f682b8468f45725b7fe7a177e14f78552d9a7546 Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Sat, 6 May 2023 23:05:25 +0000 Subject: [PATCH 01/10] Export the HadError variable from the errors package. --- golox/internal/errors/errors.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/golox/internal/errors/errors.go b/golox/internal/errors/errors.go index cf6a9f0..244b9dd 100644 --- a/golox/internal/errors/errors.go +++ b/golox/internal/errors/errors.go @@ -2,7 +2,7 @@ package errors import "fmt" -var hadError = false +var HadError = false func EmitError(line int, message string) { report(line, "", message) @@ -15,9 +15,5 @@ func report(line int, where, message string) { where, message, ) - hadError = true -} - -func HadError() bool { - return hadError + HadError = true } From b00c4506001ca1ea481d6408f1c1e2e096217113 Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Sat, 6 May 2023 23:13:13 +0000 Subject: [PATCH 02/10] Wire up errors and error state. --- golox/cmd/golox/main.go | 6 ++++++ golox/internal/runner/runner.go | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/golox/cmd/golox/main.go b/golox/cmd/golox/main.go index 0014a11..246275f 100644 --- a/golox/cmd/golox/main.go +++ b/golox/cmd/golox/main.go @@ -19,6 +19,12 @@ func main() { if errors.Is(err, runner.ErrInvalidScriptFile) { fmt.Println(err) os.Exit(1) + } else if errors.Is(err, runner.ErrScriptNotRunnable) { + fmt.Println(err) + os.Exit(65) + } else if err != nil { + fmt.Printf("Unexpected error: %v\n", err) + os.Exit(1) } default: runner.RunPrompt() diff --git a/golox/internal/runner/runner.go b/golox/internal/runner/runner.go index 9ee498a..54ccd2a 100644 --- a/golox/internal/runner/runner.go +++ b/golox/internal/runner/runner.go @@ -5,9 +5,12 @@ import ( "errors" "fmt" "os" + + lerrors "github.com/AYM1607/crafting-interpreters/golox/internal/errors" ) var ErrInvalidScriptFile = errors.New("could not read script file") +var ErrScriptNotRunnable = errors.New("could not run script") func RunPrompt() { s := bufio.NewScanner(os.Stdin) @@ -15,7 +18,9 @@ func RunPrompt() { for s.Scan() { line := s.Text() Run(line) - // TODO: resed hadError wherever it is set. + // TODO: Understand the implications of this. The book implies that it's + // to allow the users to keep issuing commands even if they make a mistake. + lerrors.HadError = false fmt.Print("> ") } } @@ -26,7 +31,9 @@ func RunFile(path string) error { return errors.Join(ErrInvalidScriptFile, err) } Run(string(fBytes)) - // TODO: check hadError and exit with a 65 code if so. + if lerrors.HadError { + return ErrScriptNotRunnable + } return nil } From 75ad2792f6788e1c34a2b7ae9e0ef8e41a6d249c Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Sat, 6 May 2023 23:21:11 +0000 Subject: [PATCH 03/10] Report invalid characters. --- golox/internal/runner/scanner.go | 6 ++++++ golox/test-invalid.lox | 1 + 2 files changed, 7 insertions(+) create mode 100644 golox/test-invalid.lox diff --git a/golox/internal/runner/scanner.go b/golox/internal/runner/scanner.go index 4d114a7..ad3b365 100644 --- a/golox/internal/runner/scanner.go +++ b/golox/internal/runner/scanner.go @@ -1,5 +1,9 @@ package runner +import ( + lerrors "github.com/AYM1607/crafting-interpreters/golox/internal/errors" +) + type Scanner struct { source string @@ -54,6 +58,8 @@ func (s *Scanner) scanToken() { s.addToken(SEMI) case '*': s.addToken(STAR) + default: + lerrors.EmitError(s.line, "Unexpected character.") } } diff --git a/golox/test-invalid.lox b/golox/test-invalid.lox new file mode 100644 index 0000000..a4f056a --- /dev/null +++ b/golox/test-invalid.lox @@ -0,0 +1 @@ +{,{.(;)-}}*@ From 021216c94a38bda33d2638fc0e3dc26ef58df9d0 Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Sat, 6 May 2023 23:48:55 +0000 Subject: [PATCH 04/10] Two character tokens, comments, spaces and new lines. --- golox/internal/runner/scanner.go | 65 ++++++++++++++++++++++++++++++++ golox/test.lox | 4 +- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/golox/internal/runner/scanner.go b/golox/internal/runner/scanner.go index ad3b365..622c74f 100644 --- a/golox/internal/runner/scanner.go +++ b/golox/internal/runner/scanner.go @@ -58,21 +58,86 @@ func (s *Scanner) scanToken() { s.addToken(SEMI) case '*': s.addToken(STAR) + case '!': + tok := BANG + if s.match('=') { + tok = BANG_EQUAL + } + s.addToken(tok) + case '=': + tok := EQUAL + if s.match('=') { + tok = EQUAL_EQUAL + } + s.addToken(tok) + case '<': + tok := LT + if s.match('=') { + tok = LTE + } + s.addToken(tok) + case '>': + tok := GT + if s.match('=') { + tok = GTE + } + s.addToken(tok) + case '/': + if s.match('/') { + // Consume all characters in a line comment. + for s.peek() != '\n' && !s.isAtEnd() { + s.advance() + } + } else { + s.addToken(SLASH) + } + // Ignore whitespace. + case ' ': + case '\t': + case '\r': + // Handle new lines. + case '\n': + s.line += 1 default: lerrors.EmitError(s.line, "Unexpected character.") } } +// advance consumes a single character from the source. func (s *Scanner) advance() byte { idx := s.current s.current += 1 return s.source[idx] } +// match returns true if the given byte is equal to the next one in source, +// it consumes the character if so. +func (s *Scanner) match(c byte) bool { + if s.isAtEnd() { + return false + } + if s.source[s.current] != c { + return false + } + + // Next character in the source matches. + s.current += 1 + return true +} + +func (s *Scanner) peek() byte { + if s.isAtEnd() { + return 0 + } + return s.source[s.current] +} + +// addToken produces a single token without a literal value. func (s *Scanner) addToken(typ TokenType) { s.addTokenWithLiteral(typ, nil) } +// addTokenWithLiteral produces a single token with the given literal value. func (s *Scanner) addTokenWithLiteral(typ TokenType, literal interface{}) { lexme := s.source[s.start:s.current] s.tokens = append( diff --git a/golox/test.lox b/golox/test.lox index da977f1..51a4006 100644 --- a/golox/test.lox +++ b/golox/test.lox @@ -1 +1,3 @@ -{,{.(;)-}}* +// this is a comment +(( )){} // grouping stuff +!*+-/=<> <= == // operators From 25e3b6068d9bf01c1f2c0e74b8badec23c79c2ad Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Sun, 7 May 2023 00:00:35 +0000 Subject: [PATCH 05/10] String literals --- golox/internal/runner/scanner.go | 24 ++++++++++++++++++++++++ golox/test.lox | 1 + 2 files changed, 25 insertions(+) diff --git a/golox/internal/runner/scanner.go b/golox/internal/runner/scanner.go index 622c74f..e5e9351 100644 --- a/golox/internal/runner/scanner.go +++ b/golox/internal/runner/scanner.go @@ -91,6 +91,8 @@ func (s *Scanner) scanToken() { } else { s.addToken(SLASH) } + case '"': + s.scanString() // Ignore whitespace. case ' ': case '\t': @@ -132,6 +134,28 @@ func (s *Scanner) peek() byte { return s.source[s.current] } +func (s *Scanner) scanString() { + for s.peek() != '"' && !s.isAtEnd() { + // Lox allows multi-line strings. + if s.peek() == '\n' { + s.line += 1 + } + s.advance() + } + + if s.isAtEnd() { + lerrors.EmitError(s.line, "Unterminated string.") + return + } + + // Consume the closing " + s.advance() + + // Trim enclosing quotes + val := s.source[s.start+1 : s.current-1] + s.addTokenWithLiteral(STRING, val) +} + // addToken produces a single token without a literal value. func (s *Scanner) addToken(typ TokenType) { s.addTokenWithLiteral(typ, nil) diff --git a/golox/test.lox b/golox/test.lox index 51a4006..a6662bd 100644 --- a/golox/test.lox +++ b/golox/test.lox @@ -1,3 +1,4 @@ // this is a comment (( )){} // grouping stuff !*+-/=<> <= == // operators +"some string literal" From cdab15193ae5c547f361c142d8c811051d10b306 Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Sun, 7 May 2023 00:28:15 +0000 Subject: [PATCH 06/10] Number literals --- golox/internal/runner/scanner.go | 45 +++++++++++++++++++++++++++ golox/internal/runner/scanner_util.go | 5 +++ 2 files changed, 50 insertions(+) create mode 100644 golox/internal/runner/scanner_util.go diff --git a/golox/internal/runner/scanner.go b/golox/internal/runner/scanner.go index e5e9351..fff253d 100644 --- a/golox/internal/runner/scanner.go +++ b/golox/internal/runner/scanner.go @@ -1,6 +1,8 @@ package runner import ( + "strconv" + lerrors "github.com/AYM1607/crafting-interpreters/golox/internal/errors" ) @@ -101,6 +103,11 @@ func (s *Scanner) scanToken() { case '\n': s.line += 1 default: + // NOTE: adding this here to avoid listing all digits in a case. + if isDigit(c) { + s.scanNumber() + return + } lerrors.EmitError(s.line, "Unexpected character.") } } @@ -134,6 +141,14 @@ func (s *Scanner) peek() byte { return s.source[s.current] } +func (s *Scanner) peekNex() byte { + idx := s.current + 1 + if idx >= len(s.source) { + return 0 + } + return s.source[idx] +} + func (s *Scanner) scanString() { for s.peek() != '"' && !s.isAtEnd() { // Lox allows multi-line strings. @@ -156,6 +171,36 @@ func (s *Scanner) scanString() { s.addTokenWithLiteral(STRING, val) } +func (s *Scanner) scanNumber() { + // Consume all digits preceding a dot (if any) + for isDigit(s.peek()) { + s.advance() + } + + // Look for a decimal part. + // Only literals in the form 123 and 123.123 are allowed. + if s.peek() == '.' && isDigit(s.peekNex()) { + // Only consume the dot if we're sure the format is valid. + s.advance() + + // Consume the rest of the digis. + for isDigit(s.peek()) { + s.advance() + } + } + // NOTE: Ignoring error because we're sure the string follows the float + // format. This should probably still report it but will leave as-is + // for now. + val, _ := strconv.ParseFloat( + s.source[s.start:s.current], + 64, + ) + s.addTokenWithLiteral( + NUMBER, + val, + ) +} + // addToken produces a single token without a literal value. func (s *Scanner) addToken(typ TokenType) { s.addTokenWithLiteral(typ, nil) diff --git a/golox/internal/runner/scanner_util.go b/golox/internal/runner/scanner_util.go new file mode 100644 index 0000000..a9e6eac --- /dev/null +++ b/golox/internal/runner/scanner_util.go @@ -0,0 +1,5 @@ +package runner + +func isDigit(c byte) bool { + return c >= '0' && c <= '9' +} From fd21194901495121ffccb21597c373e4ebf74d9d Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Sun, 7 May 2023 00:46:22 +0000 Subject: [PATCH 07/10] Keywords and identifiers --- golox/internal/runner/scanner.go | 16 ++++++++++++++++ golox/internal/runner/scanner_const.go | 20 ++++++++++++++++++++ golox/internal/runner/scanner_util.go | 10 ++++++++++ 3 files changed, 46 insertions(+) create mode 100644 golox/internal/runner/scanner_const.go diff --git a/golox/internal/runner/scanner.go b/golox/internal/runner/scanner.go index fff253d..646c042 100644 --- a/golox/internal/runner/scanner.go +++ b/golox/internal/runner/scanner.go @@ -108,6 +108,10 @@ func (s *Scanner) scanToken() { s.scanNumber() return } + if isIdentAlpha(c) { + s.scanIdentifier() + return + } lerrors.EmitError(s.line, "Unexpected character.") } } @@ -201,6 +205,18 @@ func (s *Scanner) scanNumber() { ) } +func (s *Scanner) scanIdentifier() { + for isIdentAlphaNumeric(s.peek()) { + s.advance() + } + l := s.source[s.start:s.current] + typ := IDENT + if kTyp, ok := KeywordTypes[l]; ok { + typ = kTyp + } + s.addToken(typ) +} + // addToken produces a single token without a literal value. func (s *Scanner) addToken(typ TokenType) { s.addTokenWithLiteral(typ, nil) diff --git a/golox/internal/runner/scanner_const.go b/golox/internal/runner/scanner_const.go new file mode 100644 index 0000000..aa94484 --- /dev/null +++ b/golox/internal/runner/scanner_const.go @@ -0,0 +1,20 @@ +package runner + +var KeywordTypes = map[string]TokenType{ + "and": AND, + "class": CLASS, + "else": ELSE, + "false": FALSE, + "for": FOR, + "fun": FUN, + "if": IF, + "nil": NIL, + "or": OR, + "print": PRINT, + "return": RETURN, + "super": SUPER, + "this": THIS, + "true": TRUE, + "var": VAR, + "while": WHILE, +} diff --git a/golox/internal/runner/scanner_util.go b/golox/internal/runner/scanner_util.go index a9e6eac..a7b7731 100644 --- a/golox/internal/runner/scanner_util.go +++ b/golox/internal/runner/scanner_util.go @@ -1,5 +1,15 @@ package runner +func isIdentAlphaNumeric(c byte) bool { + return isIdentAlpha(c) || isDigit(c) +} + +func isIdentAlpha(c byte) bool { + return (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + c == '_' +} + func isDigit(c byte) bool { return c >= '0' && c <= '9' } From 57739f81431c418d6c4bbf4510d74e8fd400b919 Mon Sep 17 00:00:00 2001 From: Mariano Uvalle Date: Sat, 6 May 2023 17:48:24 -0700 Subject: [PATCH 08/10] Update README.md README typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c803962..20eda18 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ # crafting-interpreters -Code for the book Crafting Interpreters by Robert Nystrom +Code for the book "Crafting Interpreters" by Robert Nystrom -Porting the Java code for the first version fo the interpreter into Go. +Porting the Java code for the first version of the interpreter into Go. From 7cdaa49a8e1d8755dbcefbf0d5adf335ef8b9612 Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Sun, 7 May 2023 01:25:59 +0000 Subject: [PATCH 09/10] Implement c-style nested comments. --- golox/internal/runner/scanner.go | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/golox/internal/runner/scanner.go b/golox/internal/runner/scanner.go index 646c042..c436550 100644 --- a/golox/internal/runner/scanner.go +++ b/golox/internal/runner/scanner.go @@ -90,6 +90,8 @@ func (s *Scanner) scanToken() { for s.peek() != '\n' && !s.isAtEnd() { s.advance() } + } else if s.match('*') { + s.scanInlineComment() } else { s.addToken(SLASH) } @@ -217,6 +219,38 @@ func (s *Scanner) scanIdentifier() { s.addToken(typ) } +func (s *Scanner) scanInlineComment() { + depth := 1 + closed := false + for !s.isAtEnd() && depth >= 1 { + p := s.peek() + pn := s.peekNex() + switch { + case p == '\n': + s.line += 1 + case p == '/' && pn == '*': + // Consume the extra character. + s.advance() + depth += 1 + case p == '*' && pn == '/': + // Consume the extra character. + s.advance() + depth -= 1 + if depth == 0 { + closed = true + } + } + // Always consume at least one character. + s.advance() + } + + // Only report an error if the last nested (could just be one) comment + // did not close. + if s.isAtEnd() && !closed { + lerrors.EmitError(s.line, "Unterminated comment.") + } +} + // addToken produces a single token without a literal value. func (s *Scanner) addToken(typ TokenType) { s.addTokenWithLiteral(typ, nil) From 02aef95bffdcc0d8774c6b7c36ccc6e503aa0081 Mon Sep 17 00:00:00 2001 From: AYM1607 Date: Wed, 10 May 2023 03:58:54 +0000 Subject: [PATCH 10/10] Refactor to multiple packages. --- golox/internal/runner/runner.go | 3 +- golox/internal/{runner => scanner}/scanner.go | 63 ++++++++++--------- .../scanner_util.go => scanner/util.go} | 2 +- .../scanner_const.go => types/const.go} | 2 +- golox/internal/{runner => types}/token.go | 2 +- .../internal/{runner => types}/token_type.go | 2 +- 6 files changed, 38 insertions(+), 36 deletions(-) rename golox/internal/{runner => scanner}/scanner.go (80%) rename golox/internal/{runner/scanner_util.go => scanner/util.go} (93%) rename golox/internal/{runner/scanner_const.go => types/const.go} (95%) rename golox/internal/{runner => types}/token.go (96%) rename golox/internal/{runner => types}/token_type.go (98%) diff --git a/golox/internal/runner/runner.go b/golox/internal/runner/runner.go index 54ccd2a..17bbea2 100644 --- a/golox/internal/runner/runner.go +++ b/golox/internal/runner/runner.go @@ -7,6 +7,7 @@ import ( "os" lerrors "github.com/AYM1607/crafting-interpreters/golox/internal/errors" + "github.com/AYM1607/crafting-interpreters/golox/internal/scanner" ) var ErrInvalidScriptFile = errors.New("could not read script file") @@ -38,7 +39,7 @@ func RunFile(path string) error { } func Run(source string) { - s := NewScanner(source) + s := scanner.NewScanner(source) tokens := s.ScanTokens() for _, t := range tokens { diff --git a/golox/internal/runner/scanner.go b/golox/internal/scanner/scanner.go similarity index 80% rename from golox/internal/runner/scanner.go rename to golox/internal/scanner/scanner.go index c436550..93e7415 100644 --- a/golox/internal/runner/scanner.go +++ b/golox/internal/scanner/scanner.go @@ -1,16 +1,17 @@ -package runner +package scanner import ( "strconv" lerrors "github.com/AYM1607/crafting-interpreters/golox/internal/errors" + "github.com/AYM1607/crafting-interpreters/golox/internal/types" ) type Scanner struct { source string // State. - tokens []Token + tokens []types.Token start int current int line int @@ -19,7 +20,7 @@ type Scanner struct { func NewScanner(source string) *Scanner { return &Scanner{ source: source, - tokens: []Token{}, + tokens: []types.Token{}, start: 0, current: 0, @@ -27,13 +28,13 @@ func NewScanner(source string) *Scanner { } } -func (s *Scanner) ScanTokens() []Token { +func (s *Scanner) ScanTokens() []types.Token { for !s.isAtEnd() { s.start = s.current s.scanToken() } - s.tokens = append(s.tokens, NewToken(EOF, "", nil, s.line)) + s.tokens = append(s.tokens, types.NewToken(types.EOF, "", nil, s.line)) return s.tokens } @@ -41,47 +42,47 @@ func (s *Scanner) scanToken() { c := s.advance() switch c { case '(': - s.addToken(LPAREN) + s.addToken(types.LPAREN) case ')': - s.addToken(RPAREN) + s.addToken(types.RPAREN) case '{': - s.addToken(LBRACE) + s.addToken(types.LBRACE) case '}': - s.addToken(RBRACE) + s.addToken(types.RBRACE) case ',': - s.addToken(COMMA) + s.addToken(types.COMMA) case '.': - s.addToken(DOT) + s.addToken(types.DOT) case '-': - s.addToken(MINUS) + s.addToken(types.MINUS) case '+': - s.addToken(PLUS) + s.addToken(types.PLUS) case ';': - s.addToken(SEMI) + s.addToken(types.SEMI) case '*': - s.addToken(STAR) + s.addToken(types.STAR) case '!': - tok := BANG + tok := types.BANG if s.match('=') { - tok = BANG_EQUAL + tok = types.BANG_EQUAL } s.addToken(tok) case '=': - tok := EQUAL + tok := types.EQUAL if s.match('=') { - tok = EQUAL_EQUAL + tok = types.EQUAL_EQUAL } s.addToken(tok) case '<': - tok := LT + tok := types.LT if s.match('=') { - tok = LTE + tok = types.LTE } s.addToken(tok) case '>': - tok := GT + tok := types.GT if s.match('=') { - tok = GTE + tok = types.GTE } s.addToken(tok) case '/': @@ -93,7 +94,7 @@ func (s *Scanner) scanToken() { } else if s.match('*') { s.scanInlineComment() } else { - s.addToken(SLASH) + s.addToken(types.SLASH) } case '"': s.scanString() @@ -174,7 +175,7 @@ func (s *Scanner) scanString() { // Trim enclosing quotes val := s.source[s.start+1 : s.current-1] - s.addTokenWithLiteral(STRING, val) + s.addTokenWithLiteral(types.STRING, val) } func (s *Scanner) scanNumber() { @@ -202,7 +203,7 @@ func (s *Scanner) scanNumber() { 64, ) s.addTokenWithLiteral( - NUMBER, + types.NUMBER, val, ) } @@ -212,8 +213,8 @@ func (s *Scanner) scanIdentifier() { s.advance() } l := s.source[s.start:s.current] - typ := IDENT - if kTyp, ok := KeywordTypes[l]; ok { + typ := types.IDENT + if kTyp, ok := types.KeywordTypes[l]; ok { typ = kTyp } s.addToken(typ) @@ -252,16 +253,16 @@ func (s *Scanner) scanInlineComment() { } // addToken produces a single token without a literal value. -func (s *Scanner) addToken(typ TokenType) { +func (s *Scanner) addToken(typ types.TokenType) { s.addTokenWithLiteral(typ, nil) } // addTokenWithLiteral produces a single token with the given literal value. -func (s *Scanner) addTokenWithLiteral(typ TokenType, literal interface{}) { +func (s *Scanner) addTokenWithLiteral(typ types.TokenType, literal interface{}) { lexme := s.source[s.start:s.current] s.tokens = append( s.tokens, - NewToken(typ, lexme, literal, s.line), + types.NewToken(typ, lexme, literal, s.line), ) } diff --git a/golox/internal/runner/scanner_util.go b/golox/internal/scanner/util.go similarity index 93% rename from golox/internal/runner/scanner_util.go rename to golox/internal/scanner/util.go index a7b7731..b9bad58 100644 --- a/golox/internal/runner/scanner_util.go +++ b/golox/internal/scanner/util.go @@ -1,4 +1,4 @@ -package runner +package scanner func isIdentAlphaNumeric(c byte) bool { return isIdentAlpha(c) || isDigit(c) diff --git a/golox/internal/runner/scanner_const.go b/golox/internal/types/const.go similarity index 95% rename from golox/internal/runner/scanner_const.go rename to golox/internal/types/const.go index aa94484..4339d50 100644 --- a/golox/internal/runner/scanner_const.go +++ b/golox/internal/types/const.go @@ -1,4 +1,4 @@ -package runner +package types var KeywordTypes = map[string]TokenType{ "and": AND, diff --git a/golox/internal/runner/token.go b/golox/internal/types/token.go similarity index 96% rename from golox/internal/runner/token.go rename to golox/internal/types/token.go index eb37604..e04d59a 100644 --- a/golox/internal/runner/token.go +++ b/golox/internal/types/token.go @@ -1,4 +1,4 @@ -package runner +package types import "fmt" diff --git a/golox/internal/runner/token_type.go b/golox/internal/types/token_type.go similarity index 98% rename from golox/internal/runner/token_type.go rename to golox/internal/types/token_type.go index 2f4950f..d2010a7 100644 --- a/golox/internal/runner/token_type.go +++ b/golox/internal/types/token_type.go @@ -1,4 +1,4 @@ -package runner +package types type TokenType string