Add a benchmark to test the Wren C API.

I've got some ideas on how to tweak the embedding API, but I want to
see what performance impact they have first, so this adds a little
benchmark that just calls a foreign method a ton of times.

Makefile

@@ -46,6 +46,10 @@ test: debug
 	@ $(MAKE) -f util/wren.mk MODE=debug test
 	@ ./util/test.py $(suite)
 
+benchmark: release
+	@ $(MAKE) -f util/wren.mk test
+	@ ./util/benchmark.py $(suite)
+
 # Generate the Wren site.
 docs:
 	@ ./util/generate_docs.py

test/api/benchmark.c

@@ -0,0 +1,21 @@
+#include <string.h>
+
+#include "benchmark.h"
+
+static void arguments(WrenVM* vm)
+{
+  double result = 0;
+  result += wrenGetArgumentDouble(vm, 1);
+  result += wrenGetArgumentDouble(vm, 2);
+  result += wrenGetArgumentDouble(vm, 3);
+  result += wrenGetArgumentDouble(vm, 4);
+  
+  wrenReturnDouble(vm, result);
+}
+
+WrenForeignMethodFn benchmarkBindMethod(const char* signature)
+{
+  if (strcmp(signature, "static Benchmark.arguments(_,_,_,_)") == 0) return arguments;
+  
+  return NULL;
+}

test/api/benchmark.h

@@ -0,0 +1,3 @@
+#include "wren.h"
+
+WrenForeignMethodFn benchmarkBindMethod(const char* signature);

test/api/call.c

@@ -5,10 +5,10 @@
 
 void callRunTests(WrenVM* vm)
 {
-  WrenValue* noParams = wrenGetMethod(vm, "main", "Api", "noParams");
+  WrenValue* noParams = wrenGetMethod(vm, "main", "Call", "noParams");
-  WrenValue* zero = wrenGetMethod(vm, "main", "Api", "zero()");
+  WrenValue* zero = wrenGetMethod(vm, "main", "Call", "zero()");
-  WrenValue* one = wrenGetMethod(vm, "main", "Api", "one(_)");
+  WrenValue* one = wrenGetMethod(vm, "main", "Call", "one(_)");
-  WrenValue* two = wrenGetMethod(vm, "main", "Api", "two(_,_)");
+  WrenValue* two = wrenGetMethod(vm, "main", "Call", "two(_,_)");
   
   // Different arity.
   wrenCall(vm, noParams, NULL, "");
@@ -16,7 +16,7 @@ void callRunTests(WrenVM* vm)
   wrenCall(vm, one, NULL, "i", 1);
   wrenCall(vm, two, NULL, "ii", 1, 2);
 
-  WrenValue* getValue = wrenGetMethod(vm, "main", "Api", "getValue(_)");
+  WrenValue* getValue = wrenGetMethod(vm, "main", "Call", "getValue(_)");
   
   // Returning a value.
   WrenValue* value = NULL;

test/api/call.wren

@@ -1,4 +1,4 @@
-class Api {
+class Call {
   static noParams {
     System.print("noParams")
   }

test/api/foreign_class.c

@@ -79,7 +79,7 @@ static void resourceFinalize(WrenVM* vm)
 
 WrenForeignMethodFn foreignClassBindMethod(const char* signature)
 {
-  if (strcmp(signature, "static Api.finalized") == 0) return apiFinalized;
+  if (strcmp(signature, "static ForeignClass.finalized") == 0) return apiFinalized;
   if (strcmp(signature, "Counter.increment(_)") == 0) return counterIncrement;
   if (strcmp(signature, "Counter.value") == 0) return counterValue;
   if (strcmp(signature, "Point.translate(_,_,_)") == 0) return pointTranslate;

test/api/foreign_class.wren

@@ -1,4 +1,4 @@
-class Api {
+class ForeignClass {
   foreign static finalized
 }
 
@@ -64,14 +64,14 @@ var resources = [
 ]
 
 System.gc()
-System.print(Api.finalized) // expect: 0
+System.print(ForeignClass.finalized) // expect: 0
 
 resources.removeAt(-1)
 
 System.gc()
-System.print(Api.finalized) // expect: 1
+System.print(ForeignClass.finalized) // expect: 1
 
 resources.clear()
 
 System.gc()
-System.print(Api.finalized) // expect: 3
+System.print(ForeignClass.finalized) // expect: 3

test/api/main.c

@@ -4,27 +4,19 @@
 #include "vm.h"
 #include "wren.h"
 
+#include "benchmark.h"
 #include "call.h"
 #include "foreign_class.h"
 #include "returns.h"
 #include "value.h"
 
-#define REGISTER_METHOD(name, camelCase) \
-  if (strcmp(testName, #name) == 0) return camelCase##BindMethod(fullName)
-
-#define REGISTER_CLASS(name, camelCase) \
-    if (strcmp(testName, #name) == 0) \
-    { \
-      camelCase##BindClass(className, &methods); \
-    }
-
 // The name of the currently executing API test.
 const char* testName;
 
 static WrenForeignMethodFn bindForeignMethod(
     WrenVM* vm, const char* module, const char* className,
     bool isStatic, const char* signature)
-{
+{  
   if (strcmp(module, "main") != 0) return NULL;
 
   // For convenience, concatenate all of the method qualifiers into a single
@@ -35,10 +27,20 @@ static WrenForeignMethodFn bindForeignMethod(
   strcat(fullName, className);
   strcat(fullName, ".");
   strcat(fullName, signature);
-
+  
-  REGISTER_METHOD(foreign_class, foreignClass);
+  WrenForeignMethodFn method = NULL;
-  REGISTER_METHOD(returns, returns);
+  
-  REGISTER_METHOD(value, value);
+  method = benchmarkBindMethod(fullName);
+  if (method != NULL) return method;
+  
+  method = foreignClassBindMethod(fullName);
+  if (method != NULL) return method;
+  
+  method = returnsBindMethod(fullName);
+  if (method != NULL) return method;
+  
+  method = valueBindMethod(fullName);
+  if (method != NULL) return method;
 
   fprintf(stderr,
       "Unknown foreign method '%s' for test '%s'\n", fullName, testName);
@@ -52,13 +54,13 @@ static WrenForeignClassMethods bindForeignClass(
   WrenForeignClassMethods methods = { NULL, NULL };
   if (strcmp(module, "main") != 0) return methods;
 
-  REGISTER_CLASS(foreign_class, foreignClass);
+  foreignClassBindClass(className, &methods);
 
   return methods;
 }
 
 static void afterLoad(WrenVM* vm) {
-  if (strcmp(testName, "call") == 0) callRunTests(vm);
+  if (strstr(testName, "call.wren") != NULL) callRunTests(vm);
 }
 
 int main(int argc, const char* argv[])
@@ -70,14 +72,7 @@ int main(int argc, const char* argv[])
   }
 
   testName = argv[1];
-
-  // The test script is at "test/api/<test>.wren".
-  char testPath[256];
-  strcpy(testPath, "test/api/");
-  strcat(testPath, testName);
-  strcat(testPath, ".wren");
-
   setTestCallbacks(bindForeignMethod, bindForeignClass, afterLoad);
-  runFile(testPath);
+  runFile(testName);
   return 0;
 }

test/api/returns.c

@@ -39,13 +39,13 @@ static void returnBytes(WrenVM* vm)
 
 WrenForeignMethodFn returnsBindMethod(const char* signature)
 {
-  if (strcmp(signature, "static Api.implicitNull") == 0) return implicitNull;
+  if (strcmp(signature, "static Returns.implicitNull") == 0) return implicitNull;
-  if (strcmp(signature, "static Api.returnInt") == 0) return returnInt;
+  if (strcmp(signature, "static Returns.returnInt") == 0) return returnInt;
-  if (strcmp(signature, "static Api.returnFloat") == 0) return returnFloat;
+  if (strcmp(signature, "static Returns.returnFloat") == 0) return returnFloat;
-  if (strcmp(signature, "static Api.returnTrue") == 0) return returnTrue;
+  if (strcmp(signature, "static Returns.returnTrue") == 0) return returnTrue;
-  if (strcmp(signature, "static Api.returnFalse") == 0) return returnFalse;
+  if (strcmp(signature, "static Returns.returnFalse") == 0) return returnFalse;
-  if (strcmp(signature, "static Api.returnString") == 0) return returnString;
+  if (strcmp(signature, "static Returns.returnString") == 0) return returnString;
-  if (strcmp(signature, "static Api.returnBytes") == 0) return returnBytes;
+  if (strcmp(signature, "static Returns.returnBytes") == 0) return returnBytes;
 
   return NULL;
 }

test/api/returns.wren

@@ -1,4 +1,4 @@
-class Api {
+class Returns {
   foreign static implicitNull
 
   foreign static returnInt
@@ -11,13 +11,13 @@ class Api {
   foreign static returnBytes
 }
 
-System.print(Api.implicitNull == null) // expect: true
+System.print(Returns.implicitNull == null) // expect: true
 
-System.print(Api.returnInt)  // expect: 123456
+System.print(Returns.returnInt)  // expect: 123456
-System.print(Api.returnFloat) // expect: 123.456
+System.print(Returns.returnFloat) // expect: 123.456
 
-System.print(Api.returnTrue)  // expect: true
+System.print(Returns.returnTrue)  // expect: true
-System.print(Api.returnFalse) // expect: false
+System.print(Returns.returnFalse) // expect: false
 
-System.print(Api.returnString) // expect: a string
+System.print(Returns.returnString) // expect: a string
-System.print(Api.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99]
+System.print(Returns.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99]

test/api/value.c

@@ -17,8 +17,8 @@ static void getValue(WrenVM* vm)
 
 WrenForeignMethodFn valueBindMethod(const char* signature)
 {
-  if (strcmp(signature, "static Api.value=(_)") == 0) return setValue;
+  if (strcmp(signature, "static Value.value=(_)") == 0) return setValue;
-  if (strcmp(signature, "static Api.value") == 0) return getValue;
+  if (strcmp(signature, "static Value.value") == 0) return getValue;
 
   return NULL;
 }

test/api/value.wren

@@ -1,9 +1,9 @@
-class Api {
+class Value {
   foreign static value=(value)
   foreign static value
 }
 
-Api.value = ["list", "of", "strings"]
+Value.value = ["list", "of", "strings"]
 
 // Do some stuff to trigger a GC (at least when GC stress testing enabled).
 var s = "string"
@@ -11,4 +11,4 @@ for (i in 1...10) {
   s = s + " more"
 }
 
-System.print(Api.value) // expect: [list, of, strings]
+System.print(Value.value) // expect: [list, of, strings]

test/benchmark/api_foreign_method.wren

@@ -0,0 +1,20 @@
+class Benchmark {
+  foreign static arguments(a, b, c, d)
+}
+
+var start = System.clock
+var result = 0
+for (i in 1..1000000) {
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+  result = result + Benchmark.arguments(1, 2, 3, 4)
+}
+System.print(result)
+System.print("elapsed: %(System.clock - start)")

util/benchmark.py

@@ -51,6 +51,8 @@ def BENCHMARK(name, pattern):
   regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE)
   BENCHMARKS.append([name, regex, None])
 
+BENCHMARK("api_foreign_method", "100000000")
+
 BENCHMARK("binary_trees", """stretch tree of depth 13 check: -1
 8192 trees of depth 4 check: -8192
 2048 trees of depth 6 check: -2048
@@ -143,9 +145,20 @@ def standard_deviation(times):
 
 def run_trial(benchmark, language):
   """Runs one benchmark one time for one language."""
+  executable_args = language[1]
+
+  # Hackish. If the benchmark name starts with "api_", it's testing the Wren
+  # C API, so run the test_api executable which has those test methods instead
+  # of the normal Wren build.
+  if benchmark[0].startswith("api_"):
+    executable_args = [
+      os.path.join(WREN_DIR, "build", "release", "test", "wren")
+    ]
+
   args = []
-  args.extend(language[1])
+  args.extend(executable_args)
   args.append(os.path.join(BENCHMARK_DIR, benchmark[0] + language[2]))
+
   try:
     out = subprocess.check_output(args, universal_newlines=True)
   except OSError:

util/test.py

@@ -111,10 +111,6 @@ class Test:
   def run(self, app, type):
     # Invoke wren and run the test.
     test_arg = self.path
-    if type == "api test":
-      # Just pass the suite name to API tests.
-      test_arg = basename(splitext(test_arg)[0])
-
     proc = Popen([app, test_arg], stdin=PIPE, stdout=PIPE, stderr=PIPE)
 
     # If a test takes longer than two seconds, kill it.

util/xcode/wren.xcodeproj/project.pbxproj

@@ -25,6 +25,7 @@
 		29729F321BA70A620099CA20 /* io.c in Sources */ = {isa = PBXBuildFile; fileRef = 29729F2E1BA70A620099CA20 /* io.c */; };
 		29729F331BA70A620099CA20 /* io.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29729F301BA70A620099CA20 /* io.wren.inc */; };
 		2986F6D71ACF93BA00BCE26C /* wren_primitive.c in Sources */ = {isa = PBXBuildFile; fileRef = 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */; };
+		29932D511C20D8C900099DEE /* benchmark.c in Sources */ = {isa = PBXBuildFile; fileRef = 29932D4F1C20D8C900099DEE /* benchmark.c */; };
 		29A427341BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; };
 		29A427351BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; };
 		29A427361BDBE435001E6E22 /* wren_opt_meta.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */; };
@@ -103,6 +104,8 @@
 		29729F301BA70A620099CA20 /* io.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = io.wren.inc; path = ../../src/module/io.wren.inc; sourceTree = "<group>"; };
 		2986F6D51ACF93BA00BCE26C /* wren_primitive.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_primitive.c; path = ../../src/vm/wren_primitive.c; sourceTree = "<group>"; };
 		2986F6D61ACF93BA00BCE26C /* wren_primitive.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_primitive.h; path = ../../src/vm/wren_primitive.h; sourceTree = "<group>"; };
+		29932D4F1C20D8C900099DEE /* benchmark.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = benchmark.c; path = ../../test/api/benchmark.c; sourceTree = "<group>"; };
+		29932D501C20D8C900099DEE /* benchmark.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = benchmark.h; path = ../../test/api/benchmark.h; sourceTree = "<group>"; };
 		29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_opt_meta.c; path = ../../src/optional/wren_opt_meta.c; sourceTree = "<group>"; };
 		29A4272F1BDBE435001E6E22 /* wren_opt_meta.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_opt_meta.h; path = ../../src/optional/wren_opt_meta.h; sourceTree = "<group>"; };
 		29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = wren_opt_meta.wren.inc; path = ../../src/optional/wren_opt_meta.wren.inc; sourceTree = "<group>"; };
@@ -239,6 +242,8 @@
 		29D0099A1B7E394F000CE58C /* api_test */ = {
 			isa = PBXGroup;
 			children = (
+				29932D4F1C20D8C900099DEE /* benchmark.c */,
+				29932D501C20D8C900099DEE /* benchmark.h */,
 				29D009A61B7E3993000CE58C /* main.c */,
 				293D46941BB43F9900200083 /* call.c */,
 				293D46951BB43F9900200083 /* call.h */,
@@ -356,6 +361,7 @@
 				291647C81BA5EC5E006142EE /* modules.c in Sources */,
 				29DC14A11BBA2FEC008A8274 /* scheduler.c in Sources */,
 				29A427391BDBE435001E6E22 /* wren_opt_random.c in Sources */,
+				29932D511C20D8C900099DEE /* benchmark.c in Sources */,
 				29DC14A01BBA2FD6008A8274 /* timer.c in Sources */,
 				29DC149F1BBA2FCC008A8274 /* vm.c in Sources */,
 				29DC14A21BBA300A008A8274 /* wren_compiler.c in Sources */,