diff --git a/Makefile b/Makefile index 5f5633fe..c402486a 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,10 @@ test: debug @ $(MAKE) -f util/wren.mk MODE=debug test @ ./util/test.py $(suite) +benchmark: release + @ $(MAKE) -f util/wren.mk test + @ ./util/benchmark.py $(suite) + # Generate the Wren site. docs: @ ./util/generate_docs.py diff --git a/test/api/benchmark.c b/test/api/benchmark.c new file mode 100644 index 00000000..5c7f4ccb --- /dev/null +++ b/test/api/benchmark.c @@ -0,0 +1,21 @@ +#include + +#include "benchmark.h" + +static void arguments(WrenVM* vm) +{ + double result = 0; + result += wrenGetArgumentDouble(vm, 1); + result += wrenGetArgumentDouble(vm, 2); + result += wrenGetArgumentDouble(vm, 3); + result += wrenGetArgumentDouble(vm, 4); + + wrenReturnDouble(vm, result); +} + +WrenForeignMethodFn benchmarkBindMethod(const char* signature) +{ + if (strcmp(signature, "static Benchmark.arguments(_,_,_,_)") == 0) return arguments; + + return NULL; +} diff --git a/test/api/benchmark.h b/test/api/benchmark.h new file mode 100644 index 00000000..de2bf4c1 --- /dev/null +++ b/test/api/benchmark.h @@ -0,0 +1,3 @@ +#include "wren.h" + +WrenForeignMethodFn benchmarkBindMethod(const char* signature); diff --git a/test/api/call.c b/test/api/call.c index 40ccf81f..5207a456 100644 --- a/test/api/call.c +++ b/test/api/call.c @@ -5,10 +5,10 @@ void callRunTests(WrenVM* vm) { - WrenValue* noParams = wrenGetMethod(vm, "main", "Api", "noParams"); - WrenValue* zero = wrenGetMethod(vm, "main", "Api", "zero()"); - WrenValue* one = wrenGetMethod(vm, "main", "Api", "one(_)"); - WrenValue* two = wrenGetMethod(vm, "main", "Api", "two(_,_)"); + WrenValue* noParams = wrenGetMethod(vm, "main", "Call", "noParams"); + WrenValue* zero = wrenGetMethod(vm, "main", "Call", "zero()"); + WrenValue* one = wrenGetMethod(vm, "main", "Call", "one(_)"); + WrenValue* two = wrenGetMethod(vm, "main", "Call", "two(_,_)"); // Different arity. wrenCall(vm, noParams, NULL, ""); @@ -16,7 +16,7 @@ void callRunTests(WrenVM* vm) wrenCall(vm, one, NULL, "i", 1); wrenCall(vm, two, NULL, "ii", 1, 2); - WrenValue* getValue = wrenGetMethod(vm, "main", "Api", "getValue(_)"); + WrenValue* getValue = wrenGetMethod(vm, "main", "Call", "getValue(_)"); // Returning a value. WrenValue* value = NULL; diff --git a/test/api/call.wren b/test/api/call.wren index a174ae2e..35feaa5e 100644 --- a/test/api/call.wren +++ b/test/api/call.wren @@ -1,4 +1,4 @@ -class Api { +class Call { static noParams { System.print("noParams") } diff --git a/test/api/foreign_class.c b/test/api/foreign_class.c index afe7f00e..cf9fbfeb 100644 --- a/test/api/foreign_class.c +++ b/test/api/foreign_class.c @@ -79,7 +79,7 @@ static void resourceFinalize(WrenVM* vm) WrenForeignMethodFn foreignClassBindMethod(const char* signature) { - if (strcmp(signature, "static Api.finalized") == 0) return apiFinalized; + if (strcmp(signature, "static ForeignClass.finalized") == 0) return apiFinalized; if (strcmp(signature, "Counter.increment(_)") == 0) return counterIncrement; if (strcmp(signature, "Counter.value") == 0) return counterValue; if (strcmp(signature, "Point.translate(_,_,_)") == 0) return pointTranslate; diff --git a/test/api/foreign_class.wren b/test/api/foreign_class.wren index 3f878271..c0b03535 100644 --- a/test/api/foreign_class.wren +++ b/test/api/foreign_class.wren @@ -1,4 +1,4 @@ -class Api { +class ForeignClass { foreign static finalized } @@ -64,14 +64,14 @@ var resources = [ ] System.gc() -System.print(Api.finalized) // expect: 0 +System.print(ForeignClass.finalized) // expect: 0 resources.removeAt(-1) System.gc() -System.print(Api.finalized) // expect: 1 +System.print(ForeignClass.finalized) // expect: 1 resources.clear() System.gc() -System.print(Api.finalized) // expect: 3 +System.print(ForeignClass.finalized) // expect: 3 diff --git a/test/api/main.c b/test/api/main.c index 7239b7ef..7674db05 100644 --- a/test/api/main.c +++ b/test/api/main.c @@ -4,27 +4,19 @@ #include "vm.h" #include "wren.h" +#include "benchmark.h" #include "call.h" #include "foreign_class.h" #include "returns.h" #include "value.h" -#define REGISTER_METHOD(name, camelCase) \ - if (strcmp(testName, #name) == 0) return camelCase##BindMethod(fullName) - -#define REGISTER_CLASS(name, camelCase) \ - if (strcmp(testName, #name) == 0) \ - { \ - camelCase##BindClass(className, &methods); \ - } - // The name of the currently executing API test. const char* testName; static WrenForeignMethodFn bindForeignMethod( WrenVM* vm, const char* module, const char* className, bool isStatic, const char* signature) -{ +{ if (strcmp(module, "main") != 0) return NULL; // For convenience, concatenate all of the method qualifiers into a single @@ -35,10 +27,20 @@ static WrenForeignMethodFn bindForeignMethod( strcat(fullName, className); strcat(fullName, "."); strcat(fullName, signature); - - REGISTER_METHOD(foreign_class, foreignClass); - REGISTER_METHOD(returns, returns); - REGISTER_METHOD(value, value); + + WrenForeignMethodFn method = NULL; + + method = benchmarkBindMethod(fullName); + if (method != NULL) return method; + + method = foreignClassBindMethod(fullName); + if (method != NULL) return method; + + method = returnsBindMethod(fullName); + if (method != NULL) return method; + + method = valueBindMethod(fullName); + if (method != NULL) return method; fprintf(stderr, "Unknown foreign method '%s' for test '%s'\n", fullName, testName); @@ -52,13 +54,13 @@ static WrenForeignClassMethods bindForeignClass( WrenForeignClassMethods methods = { NULL, NULL }; if (strcmp(module, "main") != 0) return methods; - REGISTER_CLASS(foreign_class, foreignClass); + foreignClassBindClass(className, &methods); return methods; } static void afterLoad(WrenVM* vm) { - if (strcmp(testName, "call") == 0) callRunTests(vm); + if (strstr(testName, "call.wren") != NULL) callRunTests(vm); } int main(int argc, const char* argv[]) @@ -70,14 +72,7 @@ int main(int argc, const char* argv[]) } testName = argv[1]; - - // The test script is at "test/api/.wren". - char testPath[256]; - strcpy(testPath, "test/api/"); - strcat(testPath, testName); - strcat(testPath, ".wren"); - setTestCallbacks(bindForeignMethod, bindForeignClass, afterLoad); - runFile(testPath); + runFile(testName); return 0; } diff --git a/test/api/returns.c b/test/api/returns.c index e1ba9155..f622ded9 100644 --- a/test/api/returns.c +++ b/test/api/returns.c @@ -39,13 +39,13 @@ static void returnBytes(WrenVM* vm) WrenForeignMethodFn returnsBindMethod(const char* signature) { - if (strcmp(signature, "static Api.implicitNull") == 0) return implicitNull; - if (strcmp(signature, "static Api.returnInt") == 0) return returnInt; - if (strcmp(signature, "static Api.returnFloat") == 0) return returnFloat; - if (strcmp(signature, "static Api.returnTrue") == 0) return returnTrue; - if (strcmp(signature, "static Api.returnFalse") == 0) return returnFalse; - if (strcmp(signature, "static Api.returnString") == 0) return returnString; - if (strcmp(signature, "static Api.returnBytes") == 0) return returnBytes; + if (strcmp(signature, "static Returns.implicitNull") == 0) return implicitNull; + if (strcmp(signature, "static Returns.returnInt") == 0) return returnInt; + if (strcmp(signature, "static Returns.returnFloat") == 0) return returnFloat; + if (strcmp(signature, "static Returns.returnTrue") == 0) return returnTrue; + if (strcmp(signature, "static Returns.returnFalse") == 0) return returnFalse; + if (strcmp(signature, "static Returns.returnString") == 0) return returnString; + if (strcmp(signature, "static Returns.returnBytes") == 0) return returnBytes; return NULL; } diff --git a/test/api/returns.wren b/test/api/returns.wren index b50edfcc..16a07e5e 100644 --- a/test/api/returns.wren +++ b/test/api/returns.wren @@ -1,4 +1,4 @@ -class Api { +class Returns { foreign static implicitNull foreign static returnInt @@ -11,13 +11,13 @@ class Api { foreign static returnBytes } -System.print(Api.implicitNull == null) // expect: true +System.print(Returns.implicitNull == null) // expect: true -System.print(Api.returnInt) // expect: 123456 -System.print(Api.returnFloat) // expect: 123.456 +System.print(Returns.returnInt) // expect: 123456 +System.print(Returns.returnFloat) // expect: 123.456 -System.print(Api.returnTrue) // expect: true -System.print(Api.returnFalse) // expect: false +System.print(Returns.returnTrue) // expect: true +System.print(Returns.returnFalse) // expect: false -System.print(Api.returnString) // expect: a string -System.print(Api.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99] +System.print(Returns.returnString) // expect: a string +System.print(Returns.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99] diff --git a/test/api/value.c b/test/api/value.c index 53022054..5b7b6225 100644 --- a/test/api/value.c +++ b/test/api/value.c @@ -17,8 +17,8 @@ static void getValue(WrenVM* vm) WrenForeignMethodFn valueBindMethod(const char* signature) { - if (strcmp(signature, "static Api.value=(_)") == 0) return setValue; - if (strcmp(signature, "static Api.value") == 0) return getValue; + if (strcmp(signature, "static Value.value=(_)") == 0) return setValue; + if (strcmp(signature, "static Value.value") == 0) return getValue; return NULL; } diff --git a/test/api/value.wren b/test/api/value.wren index c348869f..65e2194a 100644 --- a/test/api/value.wren +++ b/test/api/value.wren @@ -1,9 +1,9 @@ -class Api { +class Value { foreign static value=(value) foreign static value } -Api.value = ["list", "of", "strings"] +Value.value = ["list", "of", "strings"] // Do some stuff to trigger a GC (at least when GC stress testing enabled). var s = "string" @@ -11,4 +11,4 @@ for (i in 1...10) { s = s + " more" } -System.print(Api.value) // expect: [list, of, strings] +System.print(Value.value) // expect: [list, of, strings] diff --git a/test/benchmark/api_foreign_method.wren b/test/benchmark/api_foreign_method.wren new file mode 100644 index 00000000..35a7d17a --- /dev/null +++ b/test/benchmark/api_foreign_method.wren @@ -0,0 +1,20 @@ +class Benchmark { + foreign static arguments(a, b, c, d) +} + +var start = System.clock +var result = 0 +for (i in 1..1000000) { + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) +} +System.print(result) +System.print("elapsed: %(System.clock - start)") diff --git a/util/benchmark.py b/util/benchmark.py index 9e165952..36e7952d 100755 --- a/util/benchmark.py +++ b/util/benchmark.py @@ -51,6 +51,8 @@ def BENCHMARK(name, pattern): regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE) BENCHMARKS.append([name, regex, None]) +BENCHMARK("api_foreign_method", "100000000") + BENCHMARK("binary_trees", """stretch tree of depth 13 check: -1 8192 trees of depth 4 check: -8192 2048 trees of depth 6 check: -2048 @@ -143,9 +145,20 @@ def standard_deviation(times): def run_trial(benchmark, language): """Runs one benchmark one time for one language.""" + executable_args = language[1] + + # Hackish. If the benchmark name starts with "api_", it's testing the Wren + # C API, so run the test_api executable which has those test methods instead + # of the normal Wren build. + if benchmark[0].startswith("api_"): + executable_args = [ + os.path.join(WREN_DIR, "build", "release", "test", "wren") + ] + args = [] - args.extend(language[1]) + args.extend(executable_args) args.append(os.path.join(BENCHMARK_DIR, benchmark[0] + language[2])) + try: out = subprocess.check_output(args, universal_newlines=True) except OSError: diff --git a/util/test.py b/util/test.py index fdf0eac2..8e5bc0fb 100755 --- a/util/test.py +++ b/util/test.py @@ -111,10 +111,6 @@ class Test: def run(self, app, type): # Invoke wren and run the test. test_arg = self.path - if type == "api test": - # Just pass the suite name to API tests. - test_arg = basename(splitext(test_arg)[0]) - proc = Popen([app, test_arg], stdin=PIPE, stdout=PIPE, stderr=PIPE) # If a test takes longer than two seconds, kill it. diff --git a/util/xcode/wren.xcodeproj/project.pbxproj b/util/xcode/wren.xcodeproj/project.pbxproj index e30605b3..fe7b67a4 100644 --- a/util/xcode/wren.xcodeproj/project.pbxproj +++ b/util/xcode/wren.xcodeproj/project.pbxproj @@ -25,6 +25,7 @@ 29729F321BA70A620099CA20 /* io.c in Sources */ = {isa = PBXBuildFile; fileRef = 29729F2E1BA70A620099CA20 /* io.c */; }; 29729F331BA70A620099CA20 /* io.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29729F301BA70A620099CA20 /* io.wren.inc */; }; 2986F6D71ACF93BA00BCE26C /* wren_primitive.c in Sources */ = {isa = PBXBuildFile; fileRef = 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */; }; + 29932D511C20D8C900099DEE /* benchmark.c in Sources */ = {isa = PBXBuildFile; fileRef = 29932D4F1C20D8C900099DEE /* benchmark.c */; }; 29A427341BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; }; 29A427351BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; }; 29A427361BDBE435001E6E22 /* wren_opt_meta.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */; }; @@ -103,6 +104,8 @@ 29729F301BA70A620099CA20 /* io.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = io.wren.inc; path = ../../src/module/io.wren.inc; sourceTree = ""; }; 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_primitive.c; path = ../../src/vm/wren_primitive.c; sourceTree = ""; }; 2986F6D61ACF93BA00BCE26C /* wren_primitive.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_primitive.h; path = ../../src/vm/wren_primitive.h; sourceTree = ""; }; + 29932D4F1C20D8C900099DEE /* benchmark.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = benchmark.c; path = ../../test/api/benchmark.c; sourceTree = ""; }; + 29932D501C20D8C900099DEE /* benchmark.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = benchmark.h; path = ../../test/api/benchmark.h; sourceTree = ""; }; 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_opt_meta.c; path = ../../src/optional/wren_opt_meta.c; sourceTree = ""; }; 29A4272F1BDBE435001E6E22 /* wren_opt_meta.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_opt_meta.h; path = ../../src/optional/wren_opt_meta.h; sourceTree = ""; }; 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = wren_opt_meta.wren.inc; path = ../../src/optional/wren_opt_meta.wren.inc; sourceTree = ""; }; @@ -239,6 +242,8 @@ 29D0099A1B7E394F000CE58C /* api_test */ = { isa = PBXGroup; children = ( + 29932D4F1C20D8C900099DEE /* benchmark.c */, + 29932D501C20D8C900099DEE /* benchmark.h */, 29D009A61B7E3993000CE58C /* main.c */, 293D46941BB43F9900200083 /* call.c */, 293D46951BB43F9900200083 /* call.h */, @@ -356,6 +361,7 @@ 291647C81BA5EC5E006142EE /* modules.c in Sources */, 29DC14A11BBA2FEC008A8274 /* scheduler.c in Sources */, 29A427391BDBE435001E6E22 /* wren_opt_random.c in Sources */, + 29932D511C20D8C900099DEE /* benchmark.c in Sources */, 29DC14A01BBA2FD6008A8274 /* timer.c in Sources */, 29DC149F1BBA2FCC008A8274 /* vm.c in Sources */, 29DC14A21BBA300A008A8274 /* wren_compiler.c in Sources */,