diff --git a/src/compiler.c b/src/compiler.c index 9d109324..c47b3140 100644 --- a/src/compiler.c +++ b/src/compiler.c @@ -1,11 +1,77 @@ #include +#include #include +#include + #include "compiler.h" +// Note: if you add new token types, make sure to update the arrays below. +typedef enum +{ + TOKEN_LEFT_PAREN, + TOKEN_RIGHT_PAREN, + TOKEN_LEFT_BRACKET, + TOKEN_RIGHT_BRACKET, + TOKEN_LEFT_BRACE, + TOKEN_RIGHT_BRACE, + TOKEN_COLON, + TOKEN_DOT, + TOKEN_COMMA, + TOKEN_STAR, + TOKEN_SLASH, + TOKEN_PERCENT, + TOKEN_PLUS, + TOKEN_MINUS, + TOKEN_PIPE, + TOKEN_AMP, + TOKEN_BANG, + TOKEN_EQ, + TOKEN_LT, + TOKEN_GT, + TOKEN_LTEQ, + TOKEN_GTEQ, + TOKEN_EQEQ, + TOKEN_BANGEQ, + + TOKEN_ELSE, + TOKEN_IF, + TOKEN_VAR, + + TOKEN_NAME, + TOKEN_NUMBER, + TOKEN_STRING, + + TOKEN_LINE, + + TOKEN_ERROR, + TOKEN_EOF, + + MAX_TOKEN +} TokenType; + +typedef struct Token_s +{ + TokenType type; + int start; + int end; +} Token; + typedef struct { - Buffer* source; - Token* current; + const char* source; + size_t sourceLength; + + // The index in source of the beginning of the currently-being-lexed token. + int tokenStart; + + // The position of the current character being lexed. + int currentChar; + + // The most recently lexed token. + Token current; + + // The most recently consumed/advanced token. + Token previous; // The block being compiled. Block* block; @@ -23,6 +89,8 @@ typedef struct int precedence; } InfixCompiler; +// Parsing: + /* static void block(Compiler* compiler); */ @@ -33,9 +101,24 @@ static void prefixLiteral(Compiler* compiler, Token* token); static void infixCall(Compiler* compiler, Token* token); static void infixBinaryOp(Compiler* compiler, Token* token); static TokenType peek(Compiler* compiler); -static Token* match(Compiler* compiler, TokenType expected); -static Token* consume(Compiler* compiler, TokenType expected); -static Token* advance(Compiler* compiler); +static int match(Compiler* compiler, TokenType expected); +static void consume(Compiler* compiler, TokenType expected); +static void advance(Compiler* compiler); + +// Lexing: +static void readNextToken(Compiler* compiler); +static void readName(Compiler* compiler); +static void readNumber(Compiler* compiler); +static void readString(Compiler* compiler); +static void skipWhitespace(Compiler* compiler); +static int isKeyword(Compiler* compiler, const char* keyword); +static int isName(char c); +static int isDigit(char c); +static char advanceChar(Compiler* compiler); +static char peekChar(Compiler* compiler); +static void makeToken(Compiler* compiler, TokenType type); + +// Utility: static void error(Compiler* compiler, const char* format, ...); enum @@ -79,14 +162,10 @@ CompileFn prefixCompilers[] = { NULL, // TOKEN_ELSE NULL, // TOKEN_IF NULL, // TOKEN_VAR - NULL, // TOKEN_EMBEDDED prefixLiteral, // TOKEN_NAME prefixLiteral, // TOKEN_NUMBER prefixLiteral, // TOKEN_STRING NULL, // TOKEN_LINE - NULL, // TOKEN_WHITESPACE - NULL, // TOKEN_INDENT - NULL, // TOKEN_OUTDENT NULL, // TOKEN_ERROR NULL // TOKEN_EOF }; @@ -120,25 +199,28 @@ InfixCompiler infixCompilers[] = { { NULL, PREC_NONE }, // TOKEN_ELSE { NULL, PREC_NONE }, // TOKEN_IF { NULL, PREC_NONE }, // TOKEN_VAR - { NULL, PREC_NONE }, // TOKEN_EMBEDDED { NULL, PREC_NONE }, // TOKEN_NAME { NULL, PREC_NONE }, // TOKEN_NUMBER { NULL, PREC_NONE }, // TOKEN_STRING { NULL, PREC_NONE }, // TOKEN_LINE - { NULL, PREC_NONE }, // TOKEN_WHITESPACE - { NULL, PREC_NONE }, // TOKEN_INDENT - { NULL, PREC_NONE }, // TOKEN_OUTDENT { NULL, PREC_NONE }, // TOKEN_ERROR { NULL, PREC_NONE } // TOKEN_EOF }; -Block* compile(Buffer* source, Token* tokens) +Block* compile(const char* source, size_t sourceLength) { Compiler compiler; compiler.source = source; - compiler.current = tokens; + compiler.sourceLength = sourceLength; compiler.hasError = 0; + compiler.tokenStart = 0; + compiler.currentChar = 0; + + // TODO(bob): Zero-init current token. + // Read the first token. + advance(&compiler); + compiler.block = malloc(sizeof(Block)); // TODO(bob): Hack! make variable sized. compiler.block->bytecode = malloc(sizeof(Code) * 1024); @@ -161,7 +243,7 @@ Block* compile(Buffer* source, Token* tokens) } /* -static void block(Compiler* compiler) +void block(Compiler* compiler) { consume(compiler, TOKEN_INDENT); @@ -184,7 +266,7 @@ static void block(Compiler* compiler) } */ -static void statementLike(Compiler* compiler) +void statementLike(Compiler* compiler) { /* if (match(compiler, TOKEN_IF)) @@ -230,15 +312,15 @@ static void statementLike(Compiler* compiler) consume(compiler, TOKEN_LINE); } -static void expression(Compiler* compiler) +void expression(Compiler* compiler) { compilePrecedence(compiler, PREC_LOWEST); } void compilePrecedence(Compiler* compiler, int precedence) { - Token* token = advance(compiler); - CompileFn prefix = prefixCompilers[token->type]; + advance(compiler); + CompileFn prefix = prefixCompilers[compiler->previous.type]; if (prefix == NULL) { @@ -247,17 +329,17 @@ void compilePrecedence(Compiler* compiler, int precedence) exit(1); } - prefix(compiler, token); + prefix(compiler, &compiler->previous); - while (precedence <= infixCompilers[compiler->current->type].precedence) + while (precedence <= infixCompilers[compiler->current.type].precedence) { - token = advance(compiler); - CompileFn infix = infixCompilers[token->type].fn; - infix(compiler, token); + advance(compiler); + CompileFn infix = infixCompilers[compiler->previous.type].fn; + infix(compiler, &compiler->previous); } } -static void prefixLiteral(Compiler* compiler, Token* token) +void prefixLiteral(Compiler* compiler, Token* token) { // TODO(bob): Get actual value from token! // Define a constant for the literal. @@ -274,7 +356,7 @@ static void prefixLiteral(Compiler* compiler, Token* token) compiler->block->bytecode[compiler->numCodes++] = compiler->block->numConstants - 1; } -static void infixCall(Compiler* compiler, Token* token) +void infixCall(Compiler* compiler, Token* token) { printf("infix calls not implemented\n"); exit(1); @@ -304,7 +386,7 @@ static void infixCall(Compiler* compiler, Token* token) */ } -static void infixBinaryOp(Compiler* compiler, Token* token) +void infixBinaryOp(Compiler* compiler, Token* token) { printf("infix binary ops not implemented\n"); exit(1); @@ -324,43 +406,218 @@ static void infixBinaryOp(Compiler* compiler, Token* token) */ } -static TokenType peek(Compiler* compiler) +TokenType peek(Compiler* compiler) { - return compiler->current->type; + return compiler->current.type; } -static Token* match(Compiler* compiler, TokenType expected) +// TODO(bob): Make a bool type? +int match(Compiler* compiler, TokenType expected) { - if (peek(compiler) != expected) return NULL; + if (peek(compiler) != expected) return 0; - return advance(compiler); + advance(compiler); + return 1; } -static Token* consume(Compiler* compiler, TokenType expected) +void consume(Compiler* compiler, TokenType expected) { - Token* token = advance(compiler); - if (token->type != expected) + advance(compiler); + if (compiler->previous.type != expected) { // TODO(bob): Better error. - error(compiler, "Expected %d, got %d.\n", expected, token->type); + error(compiler, "Expected %d, got %d.\n", expected, compiler->previous.type); } - - return token; } -static Token* advance(Compiler* compiler) +void advance(Compiler* compiler) { // TODO(bob): Check for EOF. - Token* token = compiler->current; - compiler->current = compiler->current->next; - return unlinkToken(token); + compiler->previous = compiler->current; + readNextToken(compiler); } -static void error(Compiler* compiler, const char* format, ...) +void readNextToken(Compiler* compiler) +{ + while (peekChar(compiler) != '\0') + { + compiler->tokenStart = compiler->currentChar; + + char c = advanceChar(compiler); + switch (c) + { + case '(': makeToken(compiler, TOKEN_LEFT_PAREN); return; + case ')': makeToken(compiler, TOKEN_RIGHT_PAREN); return; + case '[': makeToken(compiler, TOKEN_LEFT_BRACKET); return; + case ']': makeToken(compiler, TOKEN_RIGHT_BRACKET); return; + case '{': makeToken(compiler, TOKEN_LEFT_BRACE); return; + case '}': makeToken(compiler, TOKEN_RIGHT_BRACE); return; + case ':': makeToken(compiler, TOKEN_COLON); return; + case '.': makeToken(compiler, TOKEN_DOT); return; + case ',': makeToken(compiler, TOKEN_COMMA); return; + case '*': makeToken(compiler, TOKEN_STAR); return; + case '/': makeToken(compiler, TOKEN_SLASH); return; + case '%': makeToken(compiler, TOKEN_PERCENT); return; + case '+': makeToken(compiler, TOKEN_PLUS); return; + case '-': makeToken(compiler, TOKEN_MINUS); return; + case '|': makeToken(compiler, TOKEN_PIPE); return; + case '&': makeToken(compiler, TOKEN_AMP); return; + case '=': + if (peekChar(compiler) == '=') + { + advanceChar(compiler); + makeToken(compiler, TOKEN_EQEQ); + } + else + { + makeToken(compiler, TOKEN_EQ); + } + return; + + case '<': + if (peekChar(compiler) == '=') + { + advanceChar(compiler); + makeToken(compiler, TOKEN_LTEQ); + } + else + { + makeToken(compiler, TOKEN_LT); + } + return; + + case '>': + if (peekChar(compiler) == '=') + { + advanceChar(compiler); + makeToken(compiler, TOKEN_GTEQ); + } + else + { + makeToken(compiler, TOKEN_GT); + } + return; + + case '!': + if (peekChar(compiler) == '=') + { + advanceChar(compiler); + makeToken(compiler, TOKEN_BANGEQ); + } + else + { + makeToken(compiler, TOKEN_BANG); + } + return; + + case '\n': makeToken(compiler, TOKEN_LINE); return; + + case ' ': skipWhitespace(compiler); break; + case '"': readString(compiler); return; + + default: + if (isName(c)) + { + readName(compiler); + } + else if (isDigit(c)) + { + readNumber(compiler); + } + else + { + makeToken(compiler, TOKEN_ERROR); + } + return; + } + } + + // If we get here, we're out of source, so just make EOF tokens. + compiler->tokenStart = compiler->currentChar; + makeToken(compiler, TOKEN_EOF); +} + +void readName(Compiler* compiler) +{ + // TODO(bob): Handle digits and EOF. + while (isName(peekChar(compiler)) || isDigit(peekChar(compiler))) advanceChar(compiler); + + TokenType type = TOKEN_NAME; + + if (isKeyword(compiler, "else")) type = TOKEN_ELSE; + else if (isKeyword(compiler, "if")) type = TOKEN_IF; + else if (isKeyword(compiler, "var")) type = TOKEN_VAR; + + makeToken(compiler, type); +} + +int isKeyword(Compiler* compiler, const char* keyword) +{ + size_t length = compiler->currentChar - compiler->tokenStart; + size_t keywordLength = strlen(keyword); + return length == keywordLength && + strncmp(compiler->source + compiler->tokenStart, keyword, length) == 0; +} + +void readNumber(Compiler* compiler) +{ + // TODO(bob): Floating point, hex, scientific, etc. + while (isDigit(peekChar(compiler))) advanceChar(compiler); + + makeToken(compiler, TOKEN_NUMBER); +} + +void readString(Compiler* compiler) +{ + // TODO(bob): Escape sequences, EOL, EOF, etc. + while (advanceChar(compiler) != '"'); + + makeToken(compiler, TOKEN_STRING); +} + +void skipWhitespace(Compiler* compiler) +{ + while (peekChar(compiler) == ' ') advanceChar(compiler); +} + +int isName(char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; +} + +int isDigit(char c) +{ + return c >= '0' && c <= '9'; +} + +char advanceChar(Compiler* compiler) +{ + char c = peekChar(compiler); + compiler->currentChar++; + return c; +} + +char peekChar(Compiler* compiler) +{ + return compiler->source[compiler->currentChar]; +} + +void makeToken(Compiler* compiler, TokenType type) +{ + compiler->current.type = type; + compiler->current.start = compiler->tokenStart; + compiler->current.end = compiler->currentChar; +} + +void error(Compiler* compiler, const char* format, ...) { compiler->hasError = 1; printf("Compile error on '"); - printToken(compiler->source, compiler->current); + + for (int i = compiler->current.start; i < compiler->current.end; i++) + { + putchar(compiler->source[i]); + } printf("': "); diff --git a/src/compiler.h b/src/compiler.h index 27506c22..317c66e5 100644 --- a/src/compiler.h +++ b/src/compiler.h @@ -1,9 +1,8 @@ #ifndef wren_parser_h #define wren_parser_h -#include "lexer.h" #include "vm.h" -Block* compile(Buffer* source, Token* tokens); +Block* compile(const char* source, size_t sourceLength); #endif diff --git a/src/lexer.c b/src/lexer.c deleted file mode 100644 index 95f74744..00000000 --- a/src/lexer.c +++ /dev/null @@ -1,232 +0,0 @@ -#include -#include - -#include "lexer.h" - -typedef struct -{ - Buffer* source; - - int start; // The beginning of the current token. - int pos; - - Token* head; - Token* tail; -} Lexer; - -static void readName(Lexer* lexer); -static void readNumber(Lexer* lexer); -static void readString(Lexer* lexer); -static void readEmbedded(Lexer* lexer); -static void readWhitespace(Lexer* lexer); -static int isKeyword(Lexer* lexer, const char* keyword); -static int isName(char c); -static int isDigit(char c); -static char advance(Lexer* lexer); -static char peek(Lexer* lexer); -static void emitToken(Lexer* lexer, TokenType type); - -Token* tokenize(Buffer* source) -{ - Lexer lexer; - lexer.source = source; - lexer.start = 0; - lexer.pos = 0; - lexer.head = NULL; - lexer.tail = NULL; - - while (peek(&lexer) != '\0') - { - lexer.start = lexer.pos; - - char c = advance(&lexer); - switch (c) - { - case '(': emitToken(&lexer, TOKEN_LEFT_PAREN); break; - case ')': emitToken(&lexer, TOKEN_RIGHT_PAREN); break; - case '[': emitToken(&lexer, TOKEN_LEFT_BRACKET); break; - case ']': emitToken(&lexer, TOKEN_RIGHT_BRACKET); break; - case '{': emitToken(&lexer, TOKEN_LEFT_BRACE); break; - case '}': emitToken(&lexer, TOKEN_RIGHT_BRACE); break; - case ':': emitToken(&lexer, TOKEN_COLON); break; - case '.': emitToken(&lexer, TOKEN_DOT); break; - case ',': emitToken(&lexer, TOKEN_COMMA); break; - case '*': emitToken(&lexer, TOKEN_STAR); break; - case '/': emitToken(&lexer, TOKEN_SLASH); break; - case '%': emitToken(&lexer, TOKEN_PERCENT); break; - case '+': emitToken(&lexer, TOKEN_PLUS); break; - case '-': emitToken(&lexer, TOKEN_MINUS); break; - case '|': emitToken(&lexer, TOKEN_PIPE); break; - case '&': emitToken(&lexer, TOKEN_AMP); break; - case '=': - if (peek(&lexer) == '=') - { - advance(&lexer); - emitToken(&lexer, TOKEN_EQEQ); - } - else - { - emitToken(&lexer, TOKEN_EQ); - } - break; - - case '<': - if (peek(&lexer) == '=') - { - advance(&lexer); - emitToken(&lexer, TOKEN_LTEQ); - } - else - { - emitToken(&lexer, TOKEN_LT); - } - break; - - case '>': - if (peek(&lexer) == '=') - { - advance(&lexer); - emitToken(&lexer, TOKEN_GTEQ); - } - else - { - emitToken(&lexer, TOKEN_GT); - } - break; - - case '!': - if (peek(&lexer) == '=') - { - advance(&lexer); - emitToken(&lexer, TOKEN_BANGEQ); - } - else - { - emitToken(&lexer, TOKEN_BANG); - } - break; - - case '\n': emitToken(&lexer, TOKEN_LINE); break; - - case ' ': readWhitespace(&lexer); break; - case '"': readString(&lexer); break; - case '`': readEmbedded(&lexer); break; - - default: - if (isName(c)) - { - readName(&lexer); - } - else if (isDigit(c)) - { - readNumber(&lexer); - } - else - { - emitToken(&lexer, TOKEN_ERROR); - } - break; - } - } - - lexer.start = lexer.pos; - emitToken(&lexer, TOKEN_EOF); - - return lexer.head; -} - -static void readName(Lexer* lexer) -{ - // TODO(bob): Handle digits and EOF. - while (isName(peek(lexer)) || isDigit(peek(lexer))) advance(lexer); - - TokenType type = TOKEN_NAME; - - if (isKeyword(lexer, "else")) type = TOKEN_ELSE; - else if (isKeyword(lexer, "if")) type = TOKEN_IF; - else if (isKeyword(lexer, "var")) type = TOKEN_VAR; - - emitToken(lexer, type); -} - -static int isKeyword(Lexer* lexer, const char* keyword) -{ - size_t length = lexer->pos - lexer->start; - size_t keywordLength = strlen(keyword); - return length == keywordLength && - strncmp(lexer->source->bytes + lexer->start, keyword, length) == 0; -} - -static void readNumber(Lexer* lexer) -{ - // TODO(bob): Floating point, hex, scientific, etc. - while (isDigit(peek(lexer))) advance(lexer); - - emitToken(lexer, TOKEN_NUMBER); -} - -static void readString(Lexer* lexer) -{ - // TODO(bob): Escape sequences, EOL, EOF, etc. - while (advance(lexer) != '"'); - - emitToken(lexer, TOKEN_STRING); -} - -static void readEmbedded(Lexer* lexer) -{ - // TODO(bob): EOF. - while (advance(lexer) != '`'); - - emitToken(lexer, TOKEN_EMBEDDED); -} - -static void readWhitespace(Lexer* lexer) -{ - while (peek(lexer) == ' ') advance(lexer); - - emitToken(lexer, TOKEN_WHITESPACE); -} - -static int isName(char c) -{ - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; -} - -static int isDigit(char c) -{ - return c >= '0' && c <= '9'; -} - -static char advance(Lexer* lexer) -{ - char c = peek(lexer); - lexer->pos++; - return c; -} - -static char peek(Lexer* lexer) -{ - return lexer->source->bytes[lexer->pos]; -} - -static void emitToken(Lexer* lexer, TokenType type) -{ - Token* token = newToken(type, lexer->start, lexer->pos); - - token->prev = lexer->tail; - token->next = NULL; - - if (lexer->tail == NULL) - { - // First token. - lexer->head = token; - } - else - { - // Not the first token, so add it to the end. - lexer->tail->next = token; - } - - lexer->tail = token; -} diff --git a/src/lexer.h b/src/lexer.h deleted file mode 100644 index d13523a7..00000000 --- a/src/lexer.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef wren_lexer_h -#define wren_lexer_h - -#include "token.h" - -Token* tokenize(Buffer* source); - -#endif diff --git a/src/main.c b/src/main.c index 1b15ddd5..63d8cc3d 100644 --- a/src/main.c +++ b/src/main.c @@ -2,22 +2,19 @@ #include #include "compiler.h" -#include "lexer.h" #include "vm.h" #define MAX_FILE_SIZE 256 * 256 -static void dumpTokens(Buffer* buffer, Token* token); - -Buffer* readFile(const char* path) +char* readFile(const char* path, size_t* length) { FILE* file = fopen(path, "r"); // TODO(bob): Handle error. - Buffer* buffer = newBuffer(MAX_FILE_SIZE); + char* buffer = malloc(MAX_FILE_SIZE); // TODO(bob): Hacky way to read a file! - size_t read = fread(buffer->bytes, sizeof(char), MAX_FILE_SIZE, file); - buffer->bytes[read] = '\0'; + *length = fread(buffer, sizeof(char), MAX_FILE_SIZE, file); + buffer[*length] = '\0'; fclose(file); return buffer; @@ -26,26 +23,18 @@ Buffer* readFile(const char* path) int main(int argc, const char * argv[]) { // TODO(bob): Validate command line arguments. - Buffer* buffer = readFile(argv[1]); - Token* tokens = tokenize(buffer); - //printf("Raw tokens:\n"); - //dumpTokens(buffer, tokens); - //printf("Cleaned tokens:\n"); - dumpTokens(buffer, tokens); - - Block* block = compile(buffer, tokens); + size_t length; + char* source = readFile(argv[1], &length); + Block* block = compile(source, length); Fiber* fiber = newFiber(); Value value = interpret(fiber, block); printValue(value); - - // TODO(bob): Free tokens. - // TODO(bob): Free ast. - - freeBuffer(buffer); + free(source); return 0; } +/* static void dumpTokens(Buffer* buffer, Token* token) { while (token) @@ -66,4 +55,5 @@ static void dumpTokens(Buffer* buffer, Token* token) token = token->next; } printf("\n"); -} \ No newline at end of file +} +*/ diff --git a/src/token.c b/src/token.c deleted file mode 100644 index 14656bda..00000000 --- a/src/token.c +++ /dev/null @@ -1,49 +0,0 @@ -#include -#include - -#include "token.h" - -Buffer* newBuffer(size_t size) -{ - Buffer* buffer = (Buffer*)malloc(sizeof(Buffer)); - buffer->bytes = (char*)malloc(size); - buffer->size = size; - - return buffer; -} - -void freeBuffer(Buffer* buffer) -{ - free(buffer->bytes); - free(buffer); -} - -Token* newToken(TokenType type, int start, int end) -{ - Token* token = (Token*)malloc(sizeof(Token)); - token->type = type; - token->start = start; - token->end = end; - token->prev = NULL; - token->next = NULL; - return token; -} - -void printToken(Buffer* buffer, Token* token) -{ - for (int i = token->start; i < token->end; i++) - { - putchar(buffer->bytes[i]); - } -} - -Token* unlinkToken(Token* token) -{ - if (token->next) token->next->prev = token->prev; - if (token->prev) token->prev->next = token->next; - - token->next = NULL; - token->prev = NULL; - - return token; -} diff --git a/src/token.h b/src/token.h deleted file mode 100644 index aa9b9c7d..00000000 --- a/src/token.h +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef wren_token_h -#define wren_token_h - -#include - -// TODO(bob): Move somewhere else? -typedef struct Buffer_s -{ - char* bytes; - size_t size; -} Buffer; - -// Note: if you add new token types, make sure to update the parser arrays in -// parser.c. -typedef enum -{ - TOKEN_LEFT_PAREN, - TOKEN_RIGHT_PAREN, - TOKEN_LEFT_BRACKET, - TOKEN_RIGHT_BRACKET, - TOKEN_LEFT_BRACE, - TOKEN_RIGHT_BRACE, - TOKEN_COLON, - TOKEN_DOT, - TOKEN_COMMA, - TOKEN_STAR, - TOKEN_SLASH, - TOKEN_PERCENT, - TOKEN_PLUS, - TOKEN_MINUS, - TOKEN_PIPE, - TOKEN_AMP, - TOKEN_BANG, - TOKEN_EQ, - TOKEN_LT, - TOKEN_GT, - TOKEN_LTEQ, - TOKEN_GTEQ, - TOKEN_EQEQ, - TOKEN_BANGEQ, - - TOKEN_ELSE, - TOKEN_IF, - TOKEN_VAR, - - TOKEN_EMBEDDED, - TOKEN_NAME, - TOKEN_NUMBER, - TOKEN_STRING, - - TOKEN_LINE, - TOKEN_WHITESPACE, - - TOKEN_ERROR, - TOKEN_EOF, - - MAX_TOKEN -} TokenType; - -typedef struct Token_s -{ - TokenType type; - int start; - int end; - - struct Token_s* prev; - struct Token_s* next; -} Token; - -Buffer* newBuffer(size_t size); -void freeBuffer(Buffer* buffer); - -// Creates a new unlinked token. -Token* newToken(TokenType type, int start, int end); - -// Prints the verbatim source text of the token. -void printToken(Buffer* buffer, Token* token); - -// Removes the token from the list containing it. Does not free it. -Token* unlinkToken(Token* token); - -#endif diff --git a/wren.xcodeproj/project.pbxproj b/wren.xcodeproj/project.pbxproj index 420135c1..305fb443 100644 --- a/wren.xcodeproj/project.pbxproj +++ b/wren.xcodeproj/project.pbxproj @@ -7,9 +7,7 @@ objects = { /* Begin PBXBuildFile section */ - 29AB1F281816E49C004B501E /* lexer.c in Sources */ = {isa = PBXBuildFile; fileRef = 29AB1F1E1816E49C004B501E /* lexer.c */; }; 29AB1F291816E49C004B501E /* main.c in Sources */ = {isa = PBXBuildFile; fileRef = 29AB1F201816E49C004B501E /* main.c */; }; - 29AB1F2C1816E49C004B501E /* token.c in Sources */ = {isa = PBXBuildFile; fileRef = 29AB1F251816E49C004B501E /* token.c */; }; 29AB1F2F1816FA66004B501E /* vm.c in Sources */ = {isa = PBXBuildFile; fileRef = 29AB1F2E1816FA66004B501E /* vm.c */; }; 29AB1F3218170104004B501E /* compiler.c in Sources */ = {isa = PBXBuildFile; fileRef = 29AB1F3018170104004B501E /* compiler.c */; }; /* End PBXBuildFile section */ @@ -28,11 +26,7 @@ /* Begin PBXFileReference section */ 29AB1F061816E3AD004B501E /* wren */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = wren; sourceTree = BUILT_PRODUCTS_DIR; }; - 29AB1F1E1816E49C004B501E /* lexer.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = lexer.c; path = src/lexer.c; sourceTree = SOURCE_ROOT; }; - 29AB1F1F1816E49C004B501E /* lexer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lexer.h; path = src/lexer.h; sourceTree = SOURCE_ROOT; }; 29AB1F201816E49C004B501E /* main.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = main.c; path = src/main.c; sourceTree = SOURCE_ROOT; }; - 29AB1F251816E49C004B501E /* token.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = token.c; path = src/token.c; sourceTree = SOURCE_ROOT; }; - 29AB1F261816E49C004B501E /* token.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = token.h; path = src/token.h; sourceTree = SOURCE_ROOT; }; 29AB1F271816E49C004B501E /* wren.1 */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.man; name = wren.1; path = src/wren.1; sourceTree = SOURCE_ROOT; }; 29AB1F2D1816FA5B004B501E /* vm.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = vm.h; path = src/vm.h; sourceTree = SOURCE_ROOT; }; 29AB1F2E1816FA66004B501E /* vm.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = vm.c; path = src/vm.c; sourceTree = SOURCE_ROOT; }; @@ -72,11 +66,7 @@ children = ( 29AB1F3018170104004B501E /* compiler.c */, 29AB1F3118170104004B501E /* compiler.h */, - 29AB1F1E1816E49C004B501E /* lexer.c */, - 29AB1F1F1816E49C004B501E /* lexer.h */, 29AB1F201816E49C004B501E /* main.c */, - 29AB1F251816E49C004B501E /* token.c */, - 29AB1F261816E49C004B501E /* token.h */, 29AB1F2E1816FA66004B501E /* vm.c */, 29AB1F2D1816FA5B004B501E /* vm.h */, 29AB1F271816E49C004B501E /* wren.1 */, @@ -139,8 +129,6 @@ 29AB1F291816E49C004B501E /* main.c in Sources */, 29AB1F3218170104004B501E /* compiler.c in Sources */, 29AB1F2F1816FA66004B501E /* vm.c in Sources */, - 29AB1F2C1816E49C004B501E /* token.c in Sources */, - 29AB1F281816E49C004B501E /* lexer.c in Sources */, ); runOnlyForDeploymentPostprocessing = 0; };