mirror of
https://github.com/wren-lang/wren.git
synced 2026-01-11 14:18:42 +01:00
Merge branch 'next-token' into main
This commit is contained in:
ruby0x1
@ -173,6 +173,9 @@ typedef struct
|
||||
// The 1-based line number of [currentChar].
|
||||
int currentLine;
|
||||
|
||||
// The upcoming token.
|
||||
Token next;
|
||||
|
||||
// The most recently lexed token.
|
||||
Token current;
|
||||
|
||||
@ -639,13 +642,13 @@ static bool matchChar(Parser* parser, char c)
|
||||
// range.
|
||||
static void makeToken(Parser* parser, TokenType type)
|
||||
{
|
||||
parser->current.type = type;
|
||||
parser->current.start = parser->tokenStart;
|
||||
parser->current.length = (int)(parser->currentChar - parser->tokenStart);
|
||||
parser->current.line = parser->currentLine;
|
||||
parser->next.type = type;
|
||||
parser->next.start = parser->tokenStart;
|
||||
parser->next.length = (int)(parser->currentChar - parser->tokenStart);
|
||||
parser->next.line = parser->currentLine;
|
||||
|
||||
// Make line tokens appear on the line containing the "\n".
|
||||
if (type == TOKEN_LINE) parser->current.line--;
|
||||
if (type == TOKEN_LINE) parser->next.line--;
|
||||
}
|
||||
|
||||
// If the current character is [c], then consumes it and makes a token of type
|
||||
@ -719,17 +722,17 @@ static void makeNumber(Parser* parser, bool isHex)
|
||||
|
||||
if (isHex)
|
||||
{
|
||||
parser->current.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16));
|
||||
parser->next.value = NUM_VAL((double)strtoll(parser->tokenStart, NULL, 16));
|
||||
}
|
||||
else
|
||||
{
|
||||
parser->current.value = NUM_VAL(strtod(parser->tokenStart, NULL));
|
||||
parser->next.value = NUM_VAL(strtod(parser->tokenStart, NULL));
|
||||
}
|
||||
|
||||
if (errno == ERANGE)
|
||||
{
|
||||
lexError(parser, "Number literal was too large (%d).", sizeof(long int));
|
||||
parser->current.value = NUM_VAL(0);
|
||||
parser->next.value = NUM_VAL(0);
|
||||
}
|
||||
|
||||
// We don't check that the entire token is consumed after calling strtoll()
|
||||
@ -921,21 +924,23 @@ static void readString(Parser* parser)
|
||||
}
|
||||
}
|
||||
|
||||
parser->current.value = wrenNewStringLength(parser->vm,
|
||||
parser->next.value = wrenNewStringLength(parser->vm,
|
||||
(char*)string.data, string.count);
|
||||
|
||||
wrenByteBufferClear(parser->vm, &string);
|
||||
makeToken(parser, type);
|
||||
}
|
||||
|
||||
// Lex the next token and store it in [parser.current].
|
||||
// Lex the next token and store it in [parser.next].
|
||||
static void nextToken(Parser* parser)
|
||||
{
|
||||
parser->previous = parser->current;
|
||||
parser->current = parser->next;
|
||||
|
||||
// If we are out of tokens, don't try to tokenize any more. We *do* still
|
||||
// copy the TOKEN_EOF to previous so that code that expects it to be consumed
|
||||
// will still work.
|
||||
if (parser->next.type == TOKEN_EOF) return;
|
||||
if (parser->current.type == TOKEN_EOF) return;
|
||||
|
||||
while (peekChar(parser) != '\0')
|
||||
@ -1094,8 +1099,8 @@ static void nextToken(Parser* parser)
|
||||
// even though the source code and console output are UTF-8.
|
||||
lexError(parser, "Invalid byte 0x%x.", (uint8_t)c);
|
||||
}
|
||||
parser->current.type = TOKEN_ERROR;
|
||||
parser->current.length = 0;
|
||||
parser->next.type = TOKEN_ERROR;
|
||||
parser->next.length = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
@ -1114,6 +1119,12 @@ static TokenType peek(Compiler* compiler)
|
||||
return compiler->parser->current.type;
|
||||
}
|
||||
|
||||
// Returns the type of the current token.
|
||||
static TokenType peekNext(Compiler* compiler)
|
||||
{
|
||||
return compiler->parser->next.type;
|
||||
}
|
||||
|
||||
// Consumes the current token if its type is [expected]. Returns true if a
|
||||
// token was consumed.
|
||||
static bool match(Compiler* compiler, TokenType expected)
|
||||
@ -1162,6 +1173,12 @@ static void consumeLine(Compiler* compiler, const char* errorMessage)
|
||||
ignoreNewlines(compiler);
|
||||
}
|
||||
|
||||
static void allowLineBeforeDot(Compiler* compiler) {
|
||||
if (peek(compiler) == TOKEN_LINE && peekNext(compiler) == TOKEN_DOT) {
|
||||
nextToken(compiler->parser);
|
||||
}
|
||||
}
|
||||
|
||||
// Variables and scopes --------------------------------------------------------
|
||||
|
||||
// Emits one single-byte argument. Returns its index.
|
||||
@ -1949,6 +1966,7 @@ static void namedCall(Compiler* compiler, bool canAssign, Code instruction)
|
||||
else
|
||||
{
|
||||
methodCall(compiler, instruction, &signature);
|
||||
allowLineBeforeDot(compiler);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2145,6 +2163,8 @@ static void field(Compiler* compiler, bool canAssign)
|
||||
loadThis(compiler);
|
||||
emitByteArg(compiler, isLoad ? CODE_LOAD_FIELD : CODE_STORE_FIELD, field);
|
||||
}
|
||||
|
||||
allowLineBeforeDot(compiler);
|
||||
}
|
||||
|
||||
// Compiles a read or assignment to [variable].
|
||||
@ -2176,6 +2196,8 @@ static void bareName(Compiler* compiler, bool canAssign, Variable variable)
|
||||
|
||||
// Emit the load instruction.
|
||||
loadVariable(compiler, variable);
|
||||
|
||||
allowLineBeforeDot(compiler);
|
||||
}
|
||||
|
||||
static void staticField(Compiler* compiler, bool canAssign)
|
||||
@ -2360,6 +2382,8 @@ static void subscript(Compiler* compiler, bool canAssign)
|
||||
finishArgumentList(compiler, &signature);
|
||||
consume(compiler, TOKEN_RIGHT_BRACKET, "Expect ']' after arguments.");
|
||||
|
||||
allowLineBeforeDot(compiler);
|
||||
|
||||
if (canAssign && match(compiler, TOKEN_EQ))
|
||||
{
|
||||
signature.type = SIG_SUBSCRIPT_SETTER;
|
||||
@ -3488,19 +3512,21 @@ ObjFn* wrenCompile(WrenVM* vm, ObjModule* module, const char* source,
|
||||
parser.numParens = 0;
|
||||
|
||||
// Zero-init the current token. This will get copied to previous when
|
||||
// advance() is called below.
|
||||
parser.current.type = TOKEN_ERROR;
|
||||
parser.current.start = source;
|
||||
parser.current.length = 0;
|
||||
parser.current.line = 0;
|
||||
parser.current.value = UNDEFINED_VAL;
|
||||
// nextToken() is called below.
|
||||
parser.next.type = TOKEN_ERROR;
|
||||
parser.next.start = source;
|
||||
parser.next.length = 0;
|
||||
parser.next.line = 0;
|
||||
parser.next.value = UNDEFINED_VAL;
|
||||
|
||||
// Ignore leading newlines.
|
||||
parser.skipNewlines = true;
|
||||
parser.printErrors = printErrors;
|
||||
parser.hasError = false;
|
||||
|
||||
// Read the first token.
|
||||
// Read the first token into next
|
||||
nextToken(&parser);
|
||||
// Copy next -> current
|
||||
nextToken(&parser);
|
||||
|
||||
int numExistingVariables = module->variables.count;
|
||||
|
||||
151
test/language/chained_newline.wren
Normal file
151
test/language/chained_newline.wren
Normal file
@ -0,0 +1,151 @@
|
||||
class Test {
|
||||
construct new() {}
|
||||
test0() {
|
||||
System.print("test0")
|
||||
return this
|
||||
}
|
||||
test1() {
|
||||
System.print("test1")
|
||||
return this
|
||||
}
|
||||
test2() {
|
||||
System.print("test2")
|
||||
return this
|
||||
}
|
||||
|
||||
[index] {
|
||||
System.print("testSubscript")
|
||||
return this
|
||||
}
|
||||
}
|
||||
|
||||
class Tester {
|
||||
construct new() {
|
||||
|
||||
var test = _test = Test.new()
|
||||
|
||||
//test local access
|
||||
|
||||
test.
|
||||
test0(). // expect: test0
|
||||
test1(). // expect: test1
|
||||
test2() // expect: test2
|
||||
|
||||
test
|
||||
.test0() // expect: test0
|
||||
.test1() // expect: test1
|
||||
.test2() // expect: test2
|
||||
|
||||
test
|
||||
.test0() // expect: test0
|
||||
.test1(). // expect: test1
|
||||
test2() // expect: test2
|
||||
|
||||
test[0] // expect: testSubscript
|
||||
.test0() // expect: test0
|
||||
|
||||
test[0]. // expect: testSubscript
|
||||
test0() // expect: test0
|
||||
|
||||
//test field access
|
||||
|
||||
_test.
|
||||
test0(). // expect: test0
|
||||
test1(). // expect: test1
|
||||
test2() // expect: test2
|
||||
|
||||
_test
|
||||
.test0() // expect: test0
|
||||
.test1() // expect: test1
|
||||
.test2() // expect: test2
|
||||
|
||||
_test
|
||||
.test0(). // expect: test0
|
||||
test1(). // expect: test1
|
||||
test2() // expect: test2
|
||||
|
||||
_test[0] // expect: testSubscript
|
||||
.test0() // expect: test0
|
||||
|
||||
_test[0]. // expect: testSubscript
|
||||
test0() // expect: test0
|
||||
|
||||
}
|
||||
|
||||
getter { _test }
|
||||
method() { _test }
|
||||
|
||||
} //Tester
|
||||
|
||||
//access via methods/getter
|
||||
|
||||
var external = Tester.new()
|
||||
|
||||
external.getter.
|
||||
test0(). // expect: test0
|
||||
test1(). // expect: test1
|
||||
test2() // expect: test2
|
||||
|
||||
external.getter
|
||||
.test0() // expect: test0
|
||||
.test1() // expect: test1
|
||||
.test2() // expect: test2
|
||||
|
||||
external.getter.
|
||||
test0() // expect: test0
|
||||
.test1() // expect: test1
|
||||
.test2() // expect: test2
|
||||
|
||||
external.getter[0]. // expect: testSubscript
|
||||
test0() // expect: test0
|
||||
|
||||
external.getter[0] // expect: testSubscript
|
||||
.test0() // expect: test0
|
||||
|
||||
external.method().
|
||||
test0(). // expect: test0
|
||||
test1(). // expect: test1
|
||||
test2() // expect: test2
|
||||
|
||||
external.method()
|
||||
.test0() // expect: test0
|
||||
.test1() // expect: test1
|
||||
.test2() // expect: test2
|
||||
|
||||
external.method().
|
||||
test0() // expect: test0
|
||||
.test1(). // expect: test1
|
||||
test2() // expect: test2
|
||||
|
||||
external.method()[0]. // expect: testSubscript
|
||||
test0() // expect: test0
|
||||
|
||||
external.method()[0] // expect: testSubscript
|
||||
.test0() // expect: test0
|
||||
|
||||
|
||||
//regular access in module scope
|
||||
|
||||
var other = Test.new()
|
||||
|
||||
other.
|
||||
test0(). // expect: test0
|
||||
test1(). // expect: test1
|
||||
test2() // expect: test2
|
||||
|
||||
other
|
||||
.test0() // expect: test0
|
||||
.test1() // expect: test1
|
||||
.test2() // expect: test2
|
||||
|
||||
other
|
||||
.test0(). // expect: test0
|
||||
test1() // expect: test1
|
||||
.test2() // expect: test2
|
||||
|
||||
|
||||
other[0] // expect: testSubscript
|
||||
.test0() // expect: test0
|
||||
|
||||
other[0]. // expect: testSubscript
|
||||
test0() // expect: test0
|
||||
Reference in New Issue
Block a user