jmug/interpreter-in-go
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement strings and partial implementation of arrays (missing index expr eval)

Browse source 
Signed-off-by: jmug <u.g.a.mariano@gmail.com>
This commit is contained in:
Mariano Uvalle 2025-01-10 17:48:54 -08:00
parent a76f47a7a3
commit 59acf6b1a1
13 changed files with 388 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

29
pkg/ast/array.go Normal file
View file

@ -0,0 +1,29 @@
package ast
import (
"bytes"
"strings"
"code.jmug.me/jmug/interpreter-in-go/pkg/token"
)
type ArrayLiteral struct {
Token token.Token // The '[' token
Elements []Expression
}
func (al *ArrayLiteral) expressionNode() {}
func (al *ArrayLiteral) TokenLiteral() string {
return al.Token.Literal
}
func (al *ArrayLiteral) String() string {
var out bytes.Buffer
elements := []string{}
for _, el := range al.Elements {
elements = append(elements, el.String())
}
out.WriteString("[")
out.WriteString(strings.Join(elements, ", "))
out.WriteString("]")
return out.String()
}

21
pkg/ast/index.go Normal file
View file

@ -0,0 +1,21 @@
package ast
import (
"fmt"
"code.jmug.me/jmug/interpreter-in-go/pkg/token"
)
type IndexExpression struct {
Token token.Token // The "[" token
Left Expression
Index Expression
}
func (ie *IndexExpression) expressionNode() {}
func (ie *IndexExpression) TokenLiteral() string {
return ie.Token.Literal
}
func (ie *IndexExpression) String() string {
return fmt.Sprintf("(%s[%s])", ie.Left.String(), ie.Index.String())
}

16
pkg/ast/string.go Normal file
View file

@ -0,0 +1,16 @@
package ast
import "code.jmug.me/jmug/interpreter-in-go/pkg/token"
type StringLiteral struct {
Token token.Token
Value string
}
func (s *StringLiteral) expressionNode() {}
func (s *StringLiteral) TokenLiteral() string {
return s.Token.Literal
}
func (s *StringLiteral) String() string {
return s.Token.Literal
}

22
pkg/evaluator/builtins.go Normal file
View file

@ -0,0 +1,22 @@
package evaluator
import "code.jmug.me/jmug/interpreter-in-go/pkg/object"
var builtins = map[string]*object.Builtin{
"len": {
Fn: func(args ...object.Object) object.Object {
if len(args) != 1 {
return newError("wrong number of arguments. got=%d, want=1",
len(args))
}
switch arg := args[0].(type) {
case *object.String:
return &object.Integer{Value: int64(len(arg.Value))}
default:
return newError("argument to `len` not supported, got %s",
args[0].Type())
}
},
},
}

46
pkg/evaluator/evaluator.go
View file

@ -25,6 +25,8 @@ func Eval(node ast.Node, env *object.Environment) object.Object {
return &object.Integer{Value: node.Value}
case *ast.Boolean:
return nativeBoolToBooleanObject(node.Value)
case *ast.StringLiteral:
return &object.String{Value: node.Value}
case *ast.PrefixExpression:
right := Eval(node.Right, env)
if isError(right) {
@ -73,6 +75,12 @@ func Eval(node ast.Node, env *object.Environment) object.Object {
return args[0]
}
return applyFunction(fn, args)
case *ast.ArrayLiteral:
els := evalExpressions(node.Elements, env)
if len(els) == 1 && isError(els[1]) {
return els[0]
}
return &object.Array{Elements: els}
}
return nil
}
@ -138,6 +146,8 @@ func evalInfixExpression(op string, left, right object.Object) object.Object {
switch {
case left.Type() == object.INTEGER_OBJ && right.Type() == object.INTEGER_OBJ:
return evalIntegerInfixExpression(op, left, right)
case left.Type() == object.STRING_OBJ && right.Type() == object.STRING_OBJ:
return evalStringInfixExpression(op, left, right)
case op == "==":
return nativeBoolToBooleanObject(left == right)
case op == "!=":
@ -177,6 +187,18 @@ func evalIntegerInfixExpression(op string, left, right object.Object) object.Obj
}
}
func evalStringInfixExpression(op string, left, right object.Object) object.Object {
if op != "+" {
return newError(
"unknown operator: %s %s %s",
left.Type(), op, right.Type(),
)
}
l := left.(*object.String).Value
r := right.(*object.String).Value
return &object.String{Value: l + r}
}
func evalIfExpression(ifExp *ast.IfExpression, env *object.Environment) object.Object {
cond := Eval(ifExp.Condition, env)
if isError(cond) {
@ -191,11 +213,13 @@ func evalIfExpression(ifExp *ast.IfExpression, env *object.Environment) object.O
}
func evalIdentifier(exp *ast.Identifier, env *object.Environment) object.Object {
val, ok := env.Get(exp.Value)
if !ok {
return newError("identifier not found: " + exp.Value)
if val, ok := env.Get(exp.Value); ok {
return val
}
return val
if val, ok := builtins[exp.Value]; ok {
return val
}
return newError("identifier not found: " + exp.Value)
}
func evalExpressions(
@ -214,13 +238,15 @@ func evalExpressions(
}
func applyFunction(fnObj object.Object, args []object.Object) object.Object {
fn, ok := fnObj.(*object.Function)
if !ok {
return newError("not a function: %s", fn.Type())
switch fn := fnObj.(type) {
case *object.Function:
env := extendFunctionEnv(fn, args)
ret := Eval(fn.Body, env)
return unwrapReturnValue(ret)
case *object.Builtin:
return fn.Fn(args...)
}
env := extendFunctionEnv(fn, args)
ret := Eval(fn.Body, env)
return unwrapReturnValue(ret)
return newError("not a function: %s", fnObj.Type())
}
func extendFunctionEnv(fn *object.Function, args []object.Object) *object.Environment {

84
pkg/evaluator/evaluator_test.go
View file

@ -186,6 +186,10 @@ if (10 > 1) {
"foobar",
"identifier not found: foobar",
},
{
`"Hello" - "World"`,
"unknown operator: STRING - STRING",
},
}
for _, tt := range tests {
@ -264,6 +268,86 @@ func TestFunctionApplication(t *testing.T) {
}
}
func TestStringLiteral(t *testing.T) {
input := `"Hello World!"`
evaluated := testEval(input)
str, ok := evaluated.(*object.String)
if !ok {
t.Fatalf("object is not String. got=%T (%+v)", evaluated, evaluated)
}
if str.Value != "Hello World!" {
t.Errorf("String has wrong value. got=%q", str.Value)
}
}
func TestStringConcatenation(t *testing.T) {
input := `"Hello" + " " + "World!"`
evaluated := testEval(input)
str, ok := evaluated.(*object.String)
if !ok {
t.Fatalf("object is not String. got=%T (%+v)", evaluated, evaluated)
}
if str.Value != "Hello World!" {
t.Errorf("String has wrong value. got=%q", str.Value)
}
}
func TestBuiltinFunctions(t *testing.T) {
tests := []struct {
input string
expected any
}{
{`len("")`, 0},
{`len("four")`, 4},
{`len("hello world")`, 11},
{`len(1)`, "argument to `len` not supported, got INTEGER"},
{`len("one", "two")`, "wrong number of arguments. got=2, want=1"},
}
for _, tt := range tests {
evaluated := testEval(tt.input)
switch expected := tt.expected.(type) {
case int:
testIntegerObject(t, evaluated, int64(expected))
case string:
errObj, ok := evaluated.(*object.Error)
if !ok {
t.Errorf("object is not Error. got=%T (%+v)",
evaluated, evaluated)
continue
}
if errObj.Message != expected {
t.Errorf("wrong error message. expected=%q, got=%q",
expected, errObj.Message)
}
}
}
}
func TestArrayLiterals(t *testing.T) {
input := "[1, 2 * 2, 3 + 3]"
evaluated := testEval(input)
result, ok := evaluated.(*object.Array)
if !ok {
t.Fatalf("object is not Array. got=%T (%+v)", evaluated, evaluated)
}
if len(result.Elements) != 3 {
t.Fatalf("array has wrong num of elements. got=%d",
len(result.Elements))
}
testIntegerObject(t, result.Elements[0], 1)
testIntegerObject(t, result.Elements[1], 4)
testIntegerObject(t, result.Elements[2], 6)
}
func testNullObject(t *testing.T, obj object.Object) bool {
if obj != _NULL {
t.Errorf("object is not NULL. got=%T (%+v)", obj, obj)

17
pkg/lexer/lexer.go
View file

@ -63,6 +63,13 @@ func (l *Lexer) NextToken() token.Token {
tok = newToken(token.LBRACE, l.ch)
case '}':
tok = newToken(token.RBRACE, l.ch)
case '[':
tok = newToken(token.LBRACKET, l.ch)
case ']':
tok = newToken(token.RBRACKET, l.ch)
case '"':
tok.Type = token.STRING
tok.Literal = l.readString()
case 0:
tok.Literal = ""
tok.Type = token.EOF
@ -122,6 +129,16 @@ func (l *Lexer) readNumber() string {
return l.input[position:l.position]
}
func (l *Lexer) readString() string {
// Don't include the quotes in the literal.
position := l.position + 1
l.readChar()
for l.ch != '"' && l.ch != 0 {
l.readChar()
}
return l.input[position:l.position]
}
func (l *Lexer) skipWhitespace() {
for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
l.readChar()

11
pkg/lexer/lexer_test.go
View file

@ -26,6 +26,9 @@ if (5 < 10) {
10 == 10;
10 != 9;
"foobar"
"foo bar"
[1, 2];
`
tests := []struct {
@ -105,6 +108,14 @@ if (5 < 10) {
{token.NOT_EQ, "!="},
{token.INT, "9"},
{token.SEMICOLON, ";"},
{token.STRING, "foobar"},
{token.STRING, "foo bar"},
{token.LBRACKET, "["},
{token.INT, "1"},
{token.COMMA, ","},
{token.INT, "2"},
{token.RBRACKET, "]"},
{token.SEMICOLON, ";"},
{token.EOF, ""},
}

41
pkg/object/object.go
View file

@ -17,6 +17,9 @@ const (
RETURN_VALUE_OBJ = "RETURN"
ERROR_OBJ = "ERROR"
FUNCTION_OBJ = "FUNCTION"
STRING_OBJ = "STRING"
BUILTIN_OBJ = "BUILTIN"
ARRAY_OBJ = "ARRAY"
)
type Object interface {
@ -97,3 +100,41 @@ func (f *Function) Inspect() string {
out.WriteString(" {\n" + f.Body.String() + "\n}")
return out.String()
}
type String struct {
Value string
}
func (s *String) Type() ObjectType {
return STRING_OBJ
}
func (s *String) Inspect() string {
return s.Value
}
type BuiltinFunction func(args ...Object) Object
type Builtin struct {
Fn BuiltinFunction
}
func (b *Builtin) Type() ObjectType {
return BUILTIN_OBJ
}
func (b *Builtin) Inspect() string {
return "builtin function"
}
type Array struct {
Elements []Object
}
func (a *Array) Type() ObjectType {
return ARRAY_OBJ
}
func (a *Array) Inspect() string {
elements := []string{}
for _, el := range a.Elements {
elements = append(elements, el.Inspect())
}
return fmt.Sprintf("[%s]", strings.Join(elements, ", "))
}

31
pkg/parser/parser.go
View file

@ -40,6 +40,8 @@ func New(l *lexer.Lexer) *Parser {
p.registerPrefix(token.LPAREN, p.parseGroupedExpression)
p.registerPrefix(token.IF, p.parseIfExpression)
p.registerPrefix(token.FUNCTION, p.parseFunctionLiteral)
p.registerPrefix(token.STRING, p.parseStringLiteral)
p.registerPrefix(token.LBRACKET, p.parseArrayLiteral)
// Infix registrations
p.registerInfix(token.PLUS, p.parseInfixExpression)
p.registerInfix(token.MINUS, p.parseInfixExpression)
@ -50,6 +52,7 @@ func New(l *lexer.Lexer) *Parser {
p.registerInfix(token.EQ, p.parseInfixExpression)
p.registerInfix(token.NOT_EQ, p.parseInfixExpression)
p.registerInfix(token.LPAREN, p.parseCallExpression)
p.registerInfix(token.LBRACKET, p.parseIndexExpression)
// TODO: figure out why this can't be done from `parseProgram`
p.nextToken()
p.nextToken()
@ -277,13 +280,13 @@ func (p *Parser) parseFunctionParameters() []*ast.Identifier {
func (p *Parser) parseCallExpression(function ast.Expression) ast.Expression {
call := &ast.CallExpression{Token: p.curToken, Function: function}
call.Arguments = p.parseCallArguments()
call.Arguments = p.parseExpressionList(token.RPAREN)
return call
}
func (p *Parser) parseCallArguments() []ast.Expression {
func (p *Parser) parseExpressionList(end token.TokenType) []ast.Expression {
args := []ast.Expression{}
if p.peekTokenIs(token.RPAREN) {
if p.peekTokenIs(end) {
p.nextToken()
return args
}
@ -297,13 +300,33 @@ func (p *Parser) parseCallArguments() []ast.Expression {
p.nextToken()
args = append(args, p.parseExpression(LOWEST))
}
if !p.nextTokenIfPeekIs(token.RPAREN) {
if !p.nextTokenIfPeekIs(end) {
// TODO: Would be good to emit an error here.
return nil
}
return args
}
func (p *Parser) parseStringLiteral() ast.Expression {
return &ast.StringLiteral{Token: p.curToken, Value: p.curToken.Literal}
}
func (p *Parser) parseArrayLiteral() ast.Expression {
array := &ast.ArrayLiteral{Token: p.curToken}
array.Elements = p.parseExpressionList(token.RBRACKET)
return array
}
func (p *Parser) parseIndexExpression(left ast.Expression) ast.Expression {
ie := &ast.IndexExpression{Token: p.curToken, Left: left}
p.nextToken()
ie.Index = p.parseExpression(LOWEST)
if !p.nextTokenIfPeekIs(token.RBRACKET) {
return nil
}
return ie
}
func (p *Parser) curTokenIs(typ token.TokenType) bool {
return p.curToken.Type == typ
}

73
pkg/parser/parser_test.go
View file

@ -353,6 +353,14 @@ func TestOperatorPrecedenceParsing(t *testing.T) {
"add(a + b + c * d / f + g)",
"add((((a + b) + ((c * d) / f)) + g))",
},
{
"a * [1, 2, 3, 4][b * c] * d",
"((a * ([1, 2, 3, 4][(b * c)])) * d)",
},
{
"add(a * b[2], b[1], 2 * [1, 2][1])",
"add((a * (b[2])), (b[1]), (2 * ([1, 2][1])))",
},
}
for _, tt := range tests {
@ -628,6 +636,71 @@ func TestCallExpressionParsing(t *testing.T) {
testInfixExpression(t, exp.Arguments[2], 4, "+", 5)
}
func TestStringLiteralExpression(t *testing.T) {
input := `"hello world";`
l := lexer.New(input)
p := New(l)
program := p.ParseProgram()
checkParserErrors(t, p)
stmt := program.Statements[0].(*ast.ExpressionStatement)
literal, ok := stmt.Expression.(*ast.StringLiteral)
if !ok {
t.Fatalf("exp not *ast.StringLiteral. got=%T", stmt.Expression)
}
if literal.Value != "hello world" {
t.Errorf("literal.Value not %q. got=%q", "hello world", literal.Value)
}
}
func TestParsingArrayLiterals(t *testing.T) {
input := "[1, 2 * 2, 3 + 3]"
l := lexer.New(input)
p := New(l)
program := p.ParseProgram()
checkParserErrors(t, p)
stmt, ok := program.Statements[0].(*ast.ExpressionStatement)
array, ok := stmt.Expression.(*ast.ArrayLiteral)
if !ok {
t.Fatalf("exp not ast.ArrayLiteral. got=%T", stmt.Expression)
}
if len(array.Elements) != 3 {
t.Fatalf("len(array.Elements) not 3. got=%d", len(array.Elements))
}
testIntegerLiteral(t, array.Elements[0], 1)
testInfixExpression(t, array.Elements[1], 2, "*", 2)
testInfixExpression(t, array.Elements[2], 3, "+", 3)
}
func TestParsingIndexExpressions(t *testing.T) {
input := "myArray[1 + 1]"
l := lexer.New(input)
p := New(l)
program := p.ParseProgram()
checkParserErrors(t, p)
stmt, ok := program.Statements[0].(*ast.ExpressionStatement)
indexExp, ok := stmt.Expression.(*ast.IndexExpression)
if !ok {
t.Fatalf("exp not *ast.IndexExpression. got=%T", stmt.Expression)
}
if !testIdentifier(t, indexExp.Left, "myArray") {
return
}
if !testInfixExpression(t, indexExp.Index, 1, "+", 1) {
return
}
}
func testIdentifier(t *testing.T, exp ast.Expression, value string) bool {
ident, ok := exp.(*ast.Identifier)
if !ok {

2
pkg/parser/precedence.go
View file

@ -13,6 +13,7 @@ const (
PRODUCT // *
PREFIX // -X or !X
CALL // myFunction(X)
INDEX // array[index]
)
var precedences = map[token.TokenType]int{
@ -25,6 +26,7 @@ var precedences = map[token.TokenType]int{
token.ASTERISK: PRODUCT,
token.SLASH: PRODUCT,
token.LPAREN: CALL,
token.LBRACKET: INDEX,
}
func (p *Parser) peekPrecedence() int {

15
pkg/token/token.go
View file

@ -7,8 +7,9 @@ const (
EOF = "EOF"
// Identifiers + Literals
IDENT = "IDENT"
INT = "INT"
IDENT = "IDENT"
INT = "INT"
STRING = "STRING"
// Operators
ASSIGN = "="
@ -26,10 +27,12 @@ const (
COMMA = ","
SEMICOLON = ";"
LPAREN = "("
RPAREN = ")"
LBRACE = "{"
RBRACE = "}"
LPAREN = "("
RPAREN = ")"
LBRACE = "{"
RBRACE = "}"
LBRACKET = "["
RBRACKET = "]"
// Keywords
FUNCTION = "FUNCTION"