From da3eac9a48bd2abc550352a5faebfea14ed36ab9 Mon Sep 17 00:00:00 2001 From: "Anthony M. Cook" Date: Fri, 20 Nov 2015 12:56:27 -0600 Subject: [PATCH 1/8] Fixing a couple of typos in the classes docs. --- doc/site/classes.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/site/classes.markdown b/doc/site/classes.markdown index 26ece8e5..b6542609 100644 --- a/doc/site/classes.markdown +++ b/doc/site/classes.markdown @@ -216,7 +216,7 @@ constructors, etc. ## Fields -All state stored in instances is stored in *fields*. Each field has a named +All state stored in instances is stored in *fields*. Each field has a name that starts with an underscore. :::wren @@ -393,7 +393,7 @@ This means you can do `super` calls inside a constructor: **TODO: Integrate better into page. Should explain this before mentioning super above.** -Sometimes you want to invoke a method on yourself, but only methods defined in +Sometimes you want to invoke a method on yourself, but using methods defined in one of your [superclasses](classes.html#inheritance). You typically do this in an overridden method when you want to access the original method being overridden. From f09e75c6e8771b5da2e4287aa5ffe285f47062b1 Mon Sep 17 00:00:00 2001 From: Rohan Singh Date: Tue, 17 Nov 2015 21:54:37 -0500 Subject: [PATCH 2/8] Update Visual Studio projects so we can build again --- .gitignore | 8 ++ src/module/io.c | 8 +- util/msvc2013/wren/wren.vcxproj | 112 ++++++++++++++++++ util/msvc2013/wren/wren.vcxproj.filters | 57 +++++++++ util/msvc2013/wren_lib/wren_lib.vcxproj | 13 +- .../wren_lib/wren_lib.vcxproj.filters | 23 ++-- 6 files changed, 203 insertions(+), 18 deletions(-) create mode 100644 util/msvc2013/wren/wren.vcxproj create mode 100644 util/msvc2013/wren/wren.vcxproj.filters diff --git a/.gitignore b/.gitignore index d996ca57..d6e8391b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,14 @@ test/benchmark/baseline.txt xcuserdata/ *.xccheckout +# Allow Visual Studio project files. +!wren/ +!wren_lib/ + +# Visual Studio intermediate files. +Debug/ +Release/ + # Visual Studio cache files. ipch/ *.aps diff --git a/src/module/io.c b/src/module/io.c index 1d0a80e9..20cfed6f 100644 --- a/src/module/io.c +++ b/src/module/io.c @@ -9,6 +9,10 @@ #include +#ifdef _WIN32 + #include +#endif + static const int stdinDescriptor = 0; // Handle to Stdin.onData_(). Called when libuv provides data on stdin. @@ -111,7 +115,7 @@ void fileOpen(WrenVM* vm) uv_fs_t* request = createRequest(wrenGetArgumentValue(vm, 2)); // TODO: Allow controlling flags and modes. - uv_fs_open(getLoop(), request, path, O_RDONLY, S_IRUSR, openCallback); + uv_fs_open(getLoop(), request, path, O_RDONLY, 0, openCallback); } // Called by libuv when the stat call for size completes. @@ -171,7 +175,7 @@ static void fileReadBytesCallback(uv_fs_t* request) { if (handleRequestError(request)) return; - uv_buf_t buffer = request->bufs[0]; + uv_buf_t buffer = request->fs.info.bufs[0]; WrenValue* fiber = freeRequest(request); // TODO: Having to copy the bytes here is a drag. It would be good if Wren's diff --git a/util/msvc2013/wren/wren.vcxproj b/util/msvc2013/wren/wren.vcxproj new file mode 100644 index 00000000..62f89948 --- /dev/null +++ b/util/msvc2013/wren/wren.vcxproj @@ -0,0 +1,112 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + + {EBF43135-4A7A-400A-8F23-DF49907025AA} + Win32Proj + wren + + + + Application + true + v120 + Unicode + + + Application + false + v120 + true + Unicode + + + + + + + + + + + + + true + $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)..\..\src\include + $(SolutionDir)..\..\build\vs\$(Configuration)\;$(SolutionDir)..\..\util\deps\libuv\Release\lib\;$(VC_LibraryPath_x86);$(WindowsSDK_LibraryPath_x86) + $(SolutionDir)..\..\build\vs\$(Configuration)\ + + + false + $(SolutionDir)..\..\src\include;$(IncludePath) + $(SolutionDir)..\..\build\vs\$(Configuration)\;$(SolutionDir)..\..\util\deps\libuv\Release\lib\;$(LibraryPath) + $(SolutionDir)..\..\build\vs\$(Configuration)\ + + + + + + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ..\..\..\src;..\..\..\src\cli;..\..\..\src\module;..\..\..\include;..\..\..\util\deps\libuv\include + + + Console + true + wren_static_d.lib;libuv.lib;ws2_32.lib;psapi.lib;iphlpapi.lib;userenv.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ..\..\..\src;..\..\..\src\cli;..\..\..\src\module;..\..\..\include;..\..\..\util\deps\libuv\include + + + Console + true + true + true + wren_static.lib;libuv.lib;ws2_32.lib;psapi.lib;iphlpapi.lib;userenv.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + + + + + + + {89cf2c43-749e-4ec4-a7c3-3f22fba9b874} + + + + + + \ No newline at end of file diff --git a/util/msvc2013/wren/wren.vcxproj.filters b/util/msvc2013/wren/wren.vcxproj.filters new file mode 100644 index 00000000..e631123c --- /dev/null +++ b/util/msvc2013/wren/wren.vcxproj.filters @@ -0,0 +1,57 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + diff --git a/util/msvc2013/wren_lib/wren_lib.vcxproj b/util/msvc2013/wren_lib/wren_lib.vcxproj index 24bd9874..b5528380 100644 --- a/util/msvc2013/wren_lib/wren_lib.vcxproj +++ b/util/msvc2013/wren_lib/wren_lib.vcxproj @@ -39,12 +39,12 @@ - $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)..\..\src\include; + $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)..\..\src\include;$(SolutionDir)..\..\src\vm;$(SolutionDir)..\..\src\optional wren_static_d $(SolutionDir)..\..\build\vs\$(Configuration)\ - $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)..\..\src\include + $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)..\..\src\include;$(SolutionDir)..\..\src\vm;$(SolutionDir)..\..\src\optional wren_static $(SolutionDir)..\..\build\vs\$(Configuration)\ @@ -79,24 +79,25 @@ - - + + - - + + + diff --git a/util/msvc2013/wren_lib/wren_lib.vcxproj.filters b/util/msvc2013/wren_lib/wren_lib.vcxproj.filters index ae3cc9af..b9423fce 100644 --- a/util/msvc2013/wren_lib/wren_lib.vcxproj.filters +++ b/util/msvc2013/wren_lib/wren_lib.vcxproj.filters @@ -24,9 +24,6 @@ Source Files - - Source Files - Source Files @@ -36,10 +33,13 @@ Source Files - + Source Files - + + Source Files + + Source Files @@ -56,23 +56,26 @@ Header Files - - Header Files - Header Files Header Files - + Header Files - + Header Files Header Files + + Header Files + + + Header Files + \ No newline at end of file From 2d3a7ece405935ee4afcfd41427c46135aa1eb9f Mon Sep 17 00:00:00 2001 From: Rohan Singh Date: Wed, 18 Nov 2015 22:36:40 -0500 Subject: [PATCH 3/8] Remove reference to timer.h, add reference to wren.h --- util/msvc2013/wren/wren.vcxproj | 1 - util/msvc2013/wren/wren.vcxproj.filters | 3 --- util/msvc2013/wren_lib/wren_lib.vcxproj | 1 + util/msvc2013/wren_lib/wren_lib.vcxproj.filters | 3 +++ 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util/msvc2013/wren/wren.vcxproj b/util/msvc2013/wren/wren.vcxproj index 62f89948..7140161e 100644 --- a/util/msvc2013/wren/wren.vcxproj +++ b/util/msvc2013/wren/wren.vcxproj @@ -99,7 +99,6 @@ - diff --git a/util/msvc2013/wren/wren.vcxproj.filters b/util/msvc2013/wren/wren.vcxproj.filters index e631123c..92a02def 100644 --- a/util/msvc2013/wren/wren.vcxproj.filters +++ b/util/msvc2013/wren/wren.vcxproj.filters @@ -50,8 +50,5 @@ Header Files - - Header Files - diff --git a/util/msvc2013/wren_lib/wren_lib.vcxproj b/util/msvc2013/wren_lib/wren_lib.vcxproj index b5528380..b1da14f1 100644 --- a/util/msvc2013/wren_lib/wren_lib.vcxproj +++ b/util/msvc2013/wren_lib/wren_lib.vcxproj @@ -87,6 +87,7 @@ + diff --git a/util/msvc2013/wren_lib/wren_lib.vcxproj.filters b/util/msvc2013/wren_lib/wren_lib.vcxproj.filters index b9423fce..a4a7df27 100644 --- a/util/msvc2013/wren_lib/wren_lib.vcxproj.filters +++ b/util/msvc2013/wren_lib/wren_lib.vcxproj.filters @@ -44,6 +44,9 @@ + + Header Files + Header Files From 8d63667ce2128f8fed0a8840f4785a39d24c3d46 Mon Sep 17 00:00:00 2001 From: Rohan Singh Date: Wed, 18 Nov 2015 22:50:30 -0500 Subject: [PATCH 4/8] Store buffer info in uv_fs_t::data so we don't depend on a private libuv field for it --- src/module/io.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/module/io.c b/src/module/io.c index 20cfed6f..b3e893ee 100644 --- a/src/module/io.c +++ b/src/module/io.c @@ -13,6 +13,12 @@ #include #endif +typedef struct sFileReqData +{ + WrenValue* fiber; + uv_buf_t buffer; +} FileReqData; + static const int stdinDescriptor = 0; // Handle to Stdin.onData_(). Called when libuv provides data on stdin. @@ -81,8 +87,13 @@ static bool handleRequestError(uv_fs_t* request) // Allocates a new request that resumes [fiber] when it completes. uv_fs_t* createRequest(WrenValue* fiber) { - uv_fs_t* request = (uv_fs_t*)malloc(sizeof(uv_fs_t)); - request->data = fiber; + char* buffer = (char*)malloc(sizeof(uv_fs_t) + sizeof(FileReqData)); + uv_fs_t* request = (uv_fs_t*)buffer; + + FileReqData* reqData = (FileReqData*)(buffer + sizeof(uv_fs_t)); + reqData->fiber = fiber; + + request->data = reqData; return request; } @@ -91,7 +102,8 @@ uv_fs_t* createRequest(WrenValue* fiber) // Returns the fiber that should be resumed after [request] completes. WrenValue* freeRequest(uv_fs_t* request) { - WrenValue* fiber = (WrenValue*)request->data; + FileReqData* reqData = (FileReqData*)request->data; + WrenValue* fiber = reqData->fiber; uv_fs_req_cleanup(request); free(request); @@ -175,7 +187,9 @@ static void fileReadBytesCallback(uv_fs_t* request) { if (handleRequestError(request)) return; - uv_buf_t buffer = request->fs.info.bufs[0]; + FileReqData* reqData = (FileReqData*)request->data; + uv_buf_t buffer = reqData->buffer; + WrenValue* fiber = freeRequest(request); // TODO: Having to copy the bytes here is a drag. It would be good if Wren's @@ -194,12 +208,14 @@ void fileReadBytes(WrenVM* vm) int fd = *(int*)wrenGetArgumentForeign(vm, 0); // TODO: Assert fd != -1. - uv_buf_t buffer; - buffer.len = (size_t)wrenGetArgumentDouble(vm, 1); - buffer.base = (char*)malloc(buffer.len); + FileReqData* reqData = (FileReqData*)request->data; + size_t length = (size_t)wrenGetArgumentDouble(vm, 1); + + reqData->buffer.len = length; + reqData->buffer.base = (char*)malloc(length); // TODO: Allow passing in offset. - uv_fs_read(getLoop(), request, fd, &buffer, 1, 0, fileReadBytesCallback); + uv_fs_read(getLoop(), request, fd, &reqData->buffer, 1, 0, fileReadBytesCallback); } void fileSize(WrenVM* vm) From 2708b0c7aef41fbe232bf8dfbc219dcf5b8b093d Mon Sep 17 00:00:00 2001 From: Rohan Singh Date: Wed, 18 Nov 2015 23:23:48 -0500 Subject: [PATCH 5/8] Fix handleRequestError casting uv_fs_t::data to the wrong type --- src/module/io.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module/io.c b/src/module/io.c index b3e893ee..13cb8f86 100644 --- a/src/module/io.c +++ b/src/module/io.c @@ -77,7 +77,8 @@ static bool handleRequestError(uv_fs_t* request) { if (request->result >= 0) return false; - WrenValue* fiber = (WrenValue*)request->data; + FileReqData* reqData = (FileReqData*)request->data; + WrenValue* fiber = (WrenValue*)reqData->fiber; schedulerResumeError(fiber, uv_strerror((int)request->result)); uv_fs_req_cleanup(request); free(request); From 2125471132dfebbe149472ac6e3adfec76cf42ab Mon Sep 17 00:00:00 2001 From: Rohan Singh Date: Wed, 25 Nov 2015 19:47:13 -0500 Subject: [PATCH 6/8] Addressing feedback * Always include fcntl.h in io.c * FileReqData -> FileRequestData * Don't combine allocations * Make output paths more consistent with the docs --- src/module/io.c | 44 ++++++++++++------------- util/msvc2013/wren/wren.vcxproj | 10 +++--- util/msvc2013/wren_lib/wren_lib.vcxproj | 6 ++-- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/module/io.c b/src/module/io.c index 13cb8f86..9f3a836b 100644 --- a/src/module/io.c +++ b/src/module/io.c @@ -8,16 +8,13 @@ #include "wren.h" #include +#include -#ifdef _WIN32 - #include -#endif - -typedef struct sFileReqData +typedef struct sFileRequestData { WrenValue* fiber; uv_buf_t buffer; -} FileReqData; +} FileRequestData; static const int stdinDescriptor = 0; @@ -77,9 +74,12 @@ static bool handleRequestError(uv_fs_t* request) { if (request->result >= 0) return false; - FileReqData* reqData = (FileReqData*)request->data; - WrenValue* fiber = (WrenValue*)reqData->fiber; + FileRequestData* data = (FileRequestData*)request->data; + WrenValue* fiber = (WrenValue*)data->fiber; + schedulerResumeError(fiber, uv_strerror((int)request->result)); + + free(data); uv_fs_req_cleanup(request); free(request); return true; @@ -88,13 +88,12 @@ static bool handleRequestError(uv_fs_t* request) // Allocates a new request that resumes [fiber] when it completes. uv_fs_t* createRequest(WrenValue* fiber) { - char* buffer = (char*)malloc(sizeof(uv_fs_t) + sizeof(FileReqData)); - uv_fs_t* request = (uv_fs_t*)buffer; + uv_fs_t* request = (uv_fs_t*)malloc(sizeof(uv_fs_t)); - FileReqData* reqData = (FileReqData*)(buffer + sizeof(uv_fs_t)); - reqData->fiber = fiber; + FileRequestData* data = (FileRequestData*)malloc(sizeof(FileRequestData)); + data->fiber = fiber; - request->data = reqData; + request->data = data; return request; } @@ -103,9 +102,10 @@ uv_fs_t* createRequest(WrenValue* fiber) // Returns the fiber that should be resumed after [request] completes. WrenValue* freeRequest(uv_fs_t* request) { - FileReqData* reqData = (FileReqData*)request->data; - WrenValue* fiber = reqData->fiber; - + FileRequestData* data = (FileRequestData*)request->data; + WrenValue* fiber = data->fiber; + + free(data); uv_fs_req_cleanup(request); free(request); @@ -188,8 +188,8 @@ static void fileReadBytesCallback(uv_fs_t* request) { if (handleRequestError(request)) return; - FileReqData* reqData = (FileReqData*)request->data; - uv_buf_t buffer = reqData->buffer; + FileRequestData* data = (FileRequestData*)request->data; + uv_buf_t buffer = data->buffer; WrenValue* fiber = freeRequest(request); @@ -209,14 +209,14 @@ void fileReadBytes(WrenVM* vm) int fd = *(int*)wrenGetArgumentForeign(vm, 0); // TODO: Assert fd != -1. - FileReqData* reqData = (FileReqData*)request->data; + FileRequestData* data = (FileRequestData*)request->data; size_t length = (size_t)wrenGetArgumentDouble(vm, 1); - reqData->buffer.len = length; - reqData->buffer.base = (char*)malloc(length); + data->buffer.len = length; + data->buffer.base = (char*)malloc(length); // TODO: Allow passing in offset. - uv_fs_read(getLoop(), request, fd, &reqData->buffer, 1, 0, fileReadBytesCallback); + uv_fs_read(getLoop(), request, fd, &data->buffer, 1, 0, fileReadBytesCallback); } void fileSize(WrenVM* vm) diff --git a/util/msvc2013/wren/wren.vcxproj b/util/msvc2013/wren/wren.vcxproj index 7140161e..e225b826 100644 --- a/util/msvc2013/wren/wren.vcxproj +++ b/util/msvc2013/wren/wren.vcxproj @@ -42,14 +42,16 @@ true $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)..\..\src\include - $(SolutionDir)..\..\build\vs\$(Configuration)\;$(SolutionDir)..\..\util\deps\libuv\Release\lib\;$(VC_LibraryPath_x86);$(WindowsSDK_LibraryPath_x86) - $(SolutionDir)..\..\build\vs\$(Configuration)\ + $(SolutionDir)..\..\lib\$(Configuration)\;$(SolutionDir)..\..\util\deps\libuv\Release\lib\;$(VC_LibraryPath_x86);$(WindowsSDK_LibraryPath_x86) + $(SolutionDir)..\..\bin\$(Configuration)\ + $(SolutionDir)..\..\build\cli\$(Configuration)\ false $(SolutionDir)..\..\src\include;$(IncludePath) - $(SolutionDir)..\..\build\vs\$(Configuration)\;$(SolutionDir)..\..\util\deps\libuv\Release\lib\;$(LibraryPath) - $(SolutionDir)..\..\build\vs\$(Configuration)\ + $(SolutionDir)..\..\lib\$(Configuration)\;$(SolutionDir)..\..\util\deps\libuv\Release\lib\;$(LibraryPath) + $(SolutionDir)..\..\bin\$(Configuration)\ + $(SolutionDir)..\..\build\cli\$(Configuration)\ diff --git a/util/msvc2013/wren_lib/wren_lib.vcxproj b/util/msvc2013/wren_lib/wren_lib.vcxproj index b1da14f1..a168270e 100644 --- a/util/msvc2013/wren_lib/wren_lib.vcxproj +++ b/util/msvc2013/wren_lib/wren_lib.vcxproj @@ -41,12 +41,14 @@ $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)..\..\src\include;$(SolutionDir)..\..\src\vm;$(SolutionDir)..\..\src\optional wren_static_d - $(SolutionDir)..\..\build\vs\$(Configuration)\ + $(SolutionDir)..\..\lib\$(Configuration)\ + $(SolutionDir)..\..\build\lib\$(Configuration)\ $(VC_IncludePath);$(WindowsSDK_IncludePath);$(SolutionDir)..\..\src\include;$(SolutionDir)..\..\src\vm;$(SolutionDir)..\..\src\optional wren_static - $(SolutionDir)..\..\build\vs\$(Configuration)\ + $(SolutionDir)..\..\lib\$(Configuration)\ + $(SolutionDir)..\..\build\lib\$(Configuration)\ From cb0edd34b3c64be9417fa58a7bf42558efe422e5 Mon Sep 17 00:00:00 2001 From: Rohan Singh Date: Wed, 25 Nov 2015 19:56:58 -0500 Subject: [PATCH 7/8] Remove VS intermediate locations from gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index d6e8391b..bb1afd09 100644 --- a/.gitignore +++ b/.gitignore @@ -23,10 +23,6 @@ xcuserdata/ !wren/ !wren_lib/ -# Visual Studio intermediate files. -Debug/ -Release/ - # Visual Studio cache files. ipch/ *.aps From c46507d937492d4921b70697fe4416881d4e9164 Mon Sep 17 00:00:00 2001 From: Bob Nystrom Date: Fri, 27 Nov 2015 11:22:09 -0800 Subject: [PATCH 8/8] Fix .gitignore (and add incorrectly ignored file!) --- .gitignore | 18 +++--- util/pygments-lexer/wren/__init__.py | 82 ++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 util/pygments-lexer/wren/__init__.py diff --git a/.gitignore b/.gitignore index bb1afd09..316ac38a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,23 @@ # Build outputs. -bin/ -lib/ -wren +/bin +/lib +/wren # Intermediate files. -build/ -deps/ -.sass-cache/ +/build +/deps +/.sass-cache +*.pyc # I leave a temporary Wren script at the top level so that I can quickly test # stuff. -scratch.wren +/scratch.wren # The baseline file is machine-specific, so doesn't get checked in. -test/benchmark/baseline.txt +/test/benchmark/baseline.txt # XCode user-specific stuff. xcuserdata/ -*.xccheckout # Allow Visual Studio project files. !wren/ diff --git a/util/pygments-lexer/wren/__init__.py b/util/pygments-lexer/wren/__init__.py new file mode 100644 index 00000000..8e14238a --- /dev/null +++ b/util/pygments-lexer/wren/__init__.py @@ -0,0 +1,82 @@ +import re +from pygments import highlight +from pygments.lexers import PythonLexer +from pygments.formatters import HtmlFormatter + +from pygments.lexer import RegexLexer +from pygments.token import * + +class WrenLexer(RegexLexer): + name = 'Wren' + aliases = ['wren'] + filenames = ['*.wren'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'root': [ + # Whitespace. + (r'\s+', Text), + (r'[,\\\[\]{}]', Punctuation), + + # Push a parenthesized state so that we know the corresponding ')' + # is for a parenthesized expression and not interpolation. + (r'\(', Punctuation, ('parenthesized', 'root')), + + # In this state, we don't know whether a closing ')' is for a + # parenthesized expression or the end of an interpolation. So, do + # a non-consuming match and let the parent state (either + # 'parenthesized' or 'interpolation' decide. + (r'(?=\))', Text, '#pop'), + + # Keywords. + (r'(break|class|construct|else|for|foreign|if|import|in|is|' + r'return|static|super|var|while)\b', Keyword), + + (r'(true|false|null)\b', Keyword.Constant), + + (r'this\b', Name.Builtin), + + # Comments. + (r'/\*', Comment.Multiline, 'comment'), + (r'//.*?$', Comment.Single), + + # Names and operators. + (r'[~!$%^&*\-=+\\|/?<>\.:]+', Operator), + (r'[A-Z][a-zA-Z_0-9]+', Name.Variable.Global), + (r'__[a-zA-Z_0-9]+', Name.Variable.Class), + (r'_[a-zA-Z_0-9]+', Name.Variable.Instance), + (r'[a-z][a-zA-Z_0-9]+', Name), + + # Numbers. + (r'\d+\.\d+([eE]-?\d+)?', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + + # Strings. + (r'L?"', String, 'string'), + ], + 'comment': [ + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'.', Comment.Multiline), # All other characters. + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\[\\%0abfnrtv"\']', String.Escape), # Escape. + (r'\\x[a-fA-F0-9]{2}', String.Escape), # Byte escape. + (r'\\u[a-fA-F0-9]{4}', String.Escape), # Unicode escape. + (r'\\U[a-fA-F0-9]{8}', String.Escape), # Long Unicode escape. + + (r'%\(', String.Interpol, ('interpolation', 'root')), + (r'.', String), # All other characters. + ], + 'parenthesized': [ + # We only get to this state when we're at a ')'. + (r'\)', Punctuation, '#pop'), + ], + 'interpolation': [ + # We only get to this state when we're at a ')'. + (r'\)', String.Interpol, '#pop'), + ], + }