From 1c5ac288312cb0111b5eb15b6e234781f1f8b8bb Mon Sep 17 00:00:00 2001 From: underscorediscovery Date: Wed, 10 Jun 2020 10:12:50 -0700 Subject: [PATCH 1/3] compiler now tracks next token (in addition to current/previous) --- src/vm/wren_compiler.c | 51 ++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/src/vm/wren_compiler.c b/src/vm/wren_compiler.c index ee8eb78a..abd4ea18 100644 --- a/src/vm/wren_compiler.c +++ b/src/vm/wren_compiler.c @@ -171,6 +171,9 @@ typedef struct // The 1-based line number of [currentChar]. int currentLine; + // The upcoming token. + Token next; + // The most recently lexed token. Token current; @@ -635,13 +638,13 @@ static bool matchChar(Parser* parser, char c) // range. static void makeToken(Parser* parser, TokenType type) { - parser->current.type = type; - parser->current.start = parser->tokenStart; - parser->current.length = (int)(parser->currentChar - parser->tokenStart); - parser->current.line = parser->currentLine; + parser->next.type = type; + parser->next.start = parser->tokenStart; + parser->next.length = (int)(parser->currentChar - parser->tokenStart); + parser->next.line = parser->currentLine; // Make line tokens appear on the line containing the "\n". - if (type == TOKEN_LINE) parser->current.line--; + if (type == TOKEN_LINE) parser->next.line--; } // If the current character is [c], then consumes it and makes a token of type @@ -715,17 +718,17 @@ static void makeNumber(Parser* parser, bool isHex) if (isHex) { - parser->current.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16)); + parser->next.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16)); } else { - parser->current.value = NUM_VAL(strtod(parser->tokenStart, NULL)); + parser->next.value = NUM_VAL(strtod(parser->tokenStart, NULL)); } if (errno == ERANGE) { lexError(parser, "Number literal was too large (%d).", sizeof(long int)); - parser->current.value = NUM_VAL(0); + parser->next.value = NUM_VAL(0); } // We don't check that the entire token is consumed after calling strtoll() @@ -917,21 +920,23 @@ static void readString(Parser* parser) } } - parser->current.value = wrenNewStringLength(parser->vm, + parser->next.value = wrenNewStringLength(parser->vm, (char*)string.data, string.count); wrenByteBufferClear(parser->vm, &string); makeToken(parser, type); } -// Lex the next token and store it in [parser.current]. +// Lex the next token and store it in [parser.next]. static void nextToken(Parser* parser) { parser->previous = parser->current; + parser->current = parser->next; // If we are out of tokens, don't try to tokenize any more. We *do* still // copy the TOKEN_EOF to previous so that code that expects it to be consumed // will still work. + if (parser->next.type == TOKEN_EOF) return; if (parser->current.type == TOKEN_EOF) return; while (peekChar(parser) != '\0') @@ -1090,8 +1095,8 @@ static void nextToken(Parser* parser) // even though the source code and console output are UTF-8. lexError(parser, "Invalid byte 0x%x.", (uint8_t)c); } - parser->current.type = TOKEN_ERROR; - parser->current.length = 0; + parser->next.type = TOKEN_ERROR; + parser->next.length = 0; } return; } @@ -1110,6 +1115,12 @@ static TokenType peek(Compiler* compiler) return compiler->parser->current.type; } +// Returns the type of the current token. +static TokenType peekNext(Compiler* compiler) +{ + return compiler->parser->next.type; +} + // Consumes the current token if its type is [expected]. Returns true if a // token was consumed. static bool match(Compiler* compiler, TokenType expected) @@ -3445,19 +3456,21 @@ ObjFn* wrenCompile(WrenVM* vm, ObjModule* module, const char* source, parser.numParens = 0; // Zero-init the current token. This will get copied to previous when - // advance() is called below. - parser.current.type = TOKEN_ERROR; - parser.current.start = source; - parser.current.length = 0; - parser.current.line = 0; - parser.current.value = UNDEFINED_VAL; + // nextToken() is called below. + parser.next.type = TOKEN_ERROR; + parser.next.start = source; + parser.next.length = 0; + parser.next.line = 0; + parser.next.value = UNDEFINED_VAL; // Ignore leading newlines. parser.skipNewlines = true; parser.printErrors = printErrors; parser.hasError = false; - // Read the first token. + // Read the first token into next + nextToken(&parser); + // Copy next -> current nextToken(&parser); int numExistingVariables = module->variables.count; From 4c496c56a6bc51c5922a5ed19b6e0ec9b1b00de6 Mon Sep 17 00:00:00 2001 From: underscorediscovery Date: Wed, 10 Jun 2020 10:13:29 -0700 Subject: [PATCH 2/3] allow a newline before dot usage, for chained/fluent interfaces --- src/vm/wren_compiler.c | 11 +++ test/language/chained_newline.wren | 114 +++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 test/language/chained_newline.wren diff --git a/src/vm/wren_compiler.c b/src/vm/wren_compiler.c index abd4ea18..856ce0f0 100644 --- a/src/vm/wren_compiler.c +++ b/src/vm/wren_compiler.c @@ -1169,6 +1169,12 @@ static void consumeLine(Compiler* compiler, const char* errorMessage) ignoreNewlines(compiler); } +static void allowLineBeforeDot(Compiler* compiler) { + if (peek(compiler) == TOKEN_LINE && peekNext(compiler) == TOKEN_DOT) { + nextToken(compiler->parser); + } +} + // Variables and scopes -------------------------------------------------------- // Emits one single-byte argument. Returns its index. @@ -1956,6 +1962,7 @@ static void namedCall(Compiler* compiler, bool canAssign, Code instruction) else { methodCall(compiler, instruction, &signature); + allowLineBeforeDot(compiler); } } @@ -2152,6 +2159,8 @@ static void field(Compiler* compiler, bool canAssign) loadThis(compiler); emitByteArg(compiler, isLoad ? CODE_LOAD_FIELD : CODE_STORE_FIELD, field); } + + allowLineBeforeDot(compiler); } // Compiles a read or assignment to [variable]. @@ -2183,6 +2192,8 @@ static void bareName(Compiler* compiler, bool canAssign, Variable variable) // Emit the load instruction. loadVariable(compiler, variable); + + allowLineBeforeDot(compiler); } static void staticField(Compiler* compiler, bool canAssign) diff --git a/test/language/chained_newline.wren b/test/language/chained_newline.wren new file mode 100644 index 00000000..72dcbae1 --- /dev/null +++ b/test/language/chained_newline.wren @@ -0,0 +1,114 @@ +class Test { + construct new() {} + test0() { + System.print("test0") + return this + } + test1() { + System.print("test1") + return this + } + test2() { + System.print("test2") + return this + } +} + +class Tester { + construct new() { + + var test = _test = Test.new() + + //test local access + + test. + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + + test + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + + test + .test0() // expect: test0 + .test1(). // expect: test1 + test2() // expect: test2 + + //test field access + + _test. + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + + _test + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + + _test + .test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + + } + + getter { _test } + method() { _test } + +} //Tester + +//access via methods/getter + +var external = Tester.new() + +external.getter. + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + +external.getter + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +external.getter. + test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +external.method(). + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + +external.method() + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +external.method(). + test0() // expect: test0 + .test1(). // expect: test1 + test2() // expect: test2 + +//regular access in module scope + +var other = Test.new() + +other. + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + +other + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +other + .test0(). // expect: test0 + test1() // expect: test1 + .test2() // expect: test2 From 3c475f01ee87f732ec2f56b45c141346feeba2d6 Mon Sep 17 00:00:00 2001 From: ruby0x1 Date: Sat, 19 Sep 2020 22:03:16 -0700 Subject: [PATCH 3/3] allow newline before dot for subscript as well, and add to tests --- src/vm/wren_compiler.c | 2 + test/language/chained_newline.wren | 127 +++++++++++++++++++---------- 2 files changed, 84 insertions(+), 45 deletions(-) diff --git a/src/vm/wren_compiler.c b/src/vm/wren_compiler.c index 856ce0f0..2a43dac3 100644 --- a/src/vm/wren_compiler.c +++ b/src/vm/wren_compiler.c @@ -2378,6 +2378,8 @@ static void subscript(Compiler* compiler, bool canAssign) finishArgumentList(compiler, &signature); consume(compiler, TOKEN_RIGHT_BRACKET, "Expect ']' after arguments."); + allowLineBeforeDot(compiler); + if (canAssign && match(compiler, TOKEN_EQ)) { signature.type = SIG_SUBSCRIPT_SETTER; diff --git a/test/language/chained_newline.wren b/test/language/chained_newline.wren index 72dcbae1..bd1aaf84 100644 --- a/test/language/chained_newline.wren +++ b/test/language/chained_newline.wren @@ -12,6 +12,11 @@ class Test { System.print("test2") return this } + + [index] { + System.print("testSubscript") + return this + } } class Tester { @@ -22,36 +27,48 @@ class Tester { //test local access test. - test0(). // expect: test0 - test1(). // expect: test1 - test2() // expect: test2 + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 test - .test0() // expect: test0 - .test1() // expect: test1 - .test2() // expect: test2 + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 test - .test0() // expect: test0 - .test1(). // expect: test1 - test2() // expect: test2 + .test0() // expect: test0 + .test1(). // expect: test1 + test2() // expect: test2 + + test[0] // expect: testSubscript + .test0() // expect: test0 + + test[0]. // expect: testSubscript + test0() // expect: test0 //test field access _test. - test0(). // expect: test0 - test1(). // expect: test1 - test2() // expect: test2 + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 _test - .test0() // expect: test0 - .test1() // expect: test1 - .test2() // expect: test2 + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 _test - .test0(). // expect: test0 - test1(). // expect: test1 - test2() // expect: test2 + .test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 + + _test[0] // expect: testSubscript + .test0() // expect: test0 + + _test[0]. // expect: testSubscript + test0() // expect: test0 } @@ -65,50 +82,70 @@ class Tester { var external = Tester.new() external.getter. - test0(). // expect: test0 - test1(). // expect: test1 - test2() // expect: test2 + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 external.getter - .test0() // expect: test0 - .test1() // expect: test1 - .test2() // expect: test2 + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 external.getter. - test0() // expect: test0 - .test1() // expect: test1 - .test2() // expect: test2 + test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 + +external.getter[0]. // expect: testSubscript + test0() // expect: test0 + +external.getter[0] // expect: testSubscript + .test0() // expect: test0 external.method(). - test0(). // expect: test0 - test1(). // expect: test1 - test2() // expect: test2 + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 external.method() - .test0() // expect: test0 - .test1() // expect: test1 - .test2() // expect: test2 + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 external.method(). - test0() // expect: test0 - .test1(). // expect: test1 - test2() // expect: test2 + test0() // expect: test0 + .test1(). // expect: test1 + test2() // expect: test2 + +external.method()[0]. // expect: testSubscript + test0() // expect: test0 + +external.method()[0] // expect: testSubscript + .test0() // expect: test0 + //regular access in module scope var other = Test.new() other. - test0(). // expect: test0 - test1(). // expect: test1 - test2() // expect: test2 + test0(). // expect: test0 + test1(). // expect: test1 + test2() // expect: test2 other - .test0() // expect: test0 - .test1() // expect: test1 - .test2() // expect: test2 + .test0() // expect: test0 + .test1() // expect: test1 + .test2() // expect: test2 other - .test0(). // expect: test0 - test1() // expect: test1 - .test2() // expect: test2 + .test0(). // expect: test0 + test1() // expect: test1 + .test2() // expect: test2 + + +other[0] // expect: testSubscript + .test0() // expect: test0 + +other[0]. // expect: testSubscript + test0() // expect: test0