From 5aaaa33552b4ada4f0d0594f678f813fb54a8ae6 Mon Sep 17 00:00:00 2001
From: Bob Nystrom <robert@stuffwithstuff.com>
Date: Thu, 12 Dec 2013 16:59:57 -0800
Subject: [PATCH] Revise benchmarks:

- Ditch JS since it's in a different league.
- Make binary_trees and fib run faster.
- Compare using best time instead of mean.
---
 benchmark/binary_trees.js   | 54 ----------------------
 benchmark/binary_trees.lua  |  8 ++--
 benchmark/binary_trees.py   |  8 ++--
 benchmark/binary_trees.rb   |  8 ++--
 benchmark/binary_trees.wren | 10 ++---
 benchmark/fib.js            | 13 ------
 benchmark/fib.lua           |  2 +-
 benchmark/fib.py            |  2 +-
 benchmark/fib.rb            |  2 +-
 benchmark/fib.wren          |  2 +-
 benchmark/method_call.js    | 58 ------------------------
 benchmark/run_bench         | 90 +++++++++++++++++++++++++------------
 12 files changed, 82 insertions(+), 175 deletions(-)
 delete mode 100644 benchmark/binary_trees.js
 delete mode 100644 benchmark/fib.js
 delete mode 100644 benchmark/method_call.js

diff --git a/benchmark/binary_trees.js b/benchmark/binary_trees.js
deleted file mode 100644
index 9e63fd12..00000000
--- a/benchmark/binary_trees.js
+++ /dev/null
@@ -1,54 +0,0 @@
-/* The Great Computer Language Shootout
-http://shootout.alioth.debian.org/
-contributed by Isaac Gouy */
-
-function TreeNode(left, right, item) {
-  this.left = left;
-  this.right = right;
-  this.item = item;
-}
-
-TreeNode.prototype.itemCheck = function() {
-  if (this.left == null) return this.item;
-  return this.item + this.left.itemCheck() - this.right.itemCheck();
-}
-
-function bottomUpTree(item, depth) {
-  if (depth > 0) {
-    return new TreeNode(
-        bottomUpTree(2 * item - 1, depth - 1),
-        bottomUpTree(2 * item, depth - 1), item);
-  }
-
-  return new TreeNode(null, null, item);
-}
-
-var minDepth = 4;
-var maxDepth = 14;
-var stretchDepth = maxDepth + 1;
-
-var start = process.hrtime();
-
-var check = bottomUpTree(0, stretchDepth).itemCheck();
-console.log("stretch tree of depth " + stretchDepth + "\t check: " + check);
-
-var longLivedTree = bottomUpTree(0, maxDepth);
-for (var depth = minDepth; depth <= maxDepth; depth += 2) {
-  var iterations = 1 << (maxDepth - depth + minDepth);
-
-  check = 0;
-  for (var i = 1; i <= iterations; i++) {
-    check += bottomUpTree(i, depth).itemCheck();
-    check += bottomUpTree(-i, depth).itemCheck();
-  }
-
-  console.log(iterations * 2 + "\t trees of depth " + depth +
-      "\t check: " + check);
-}
-
-console.log("long lived tree of depth " + maxDepth + "\t check: "
-    + longLivedTree.itemCheck());
-
-var elapsed = process.hrtime(start);
-elapsed = elapsed[0] + elapsed[1] / 1000000000;
-console.log("elapsed: " + elapsed);
diff --git a/benchmark/binary_trees.lua b/benchmark/binary_trees.lua
index 2e948ddd..184cc409 100644
--- a/benchmark/binary_trees.lua
+++ b/benchmark/binary_trees.lua
@@ -21,7 +21,7 @@ local function ItemCheck(tree)
   end
 end
 
-local N = 14
+local N = 12
 local mindepth = 4
 local maxdepth = mindepth + 2
 if maxdepth < N then maxdepth = N end
@@ -31,7 +31,7 @@ local start = os.clock()
 do
   local stretchdepth = maxdepth + 1
   local stretchtree = BottomUpTree(0, stretchdepth)
-  io.write(string.format("stretch tree of depth %d\t check: %d\n",
+  io.write(string.format("stretch tree of depth %d check: %d\n",
     stretchdepth, ItemCheck(stretchtree)))
 end
 
@@ -44,11 +44,11 @@ for depth=mindepth,maxdepth,2 do
     check = check + ItemCheck(BottomUpTree(1, depth)) +
             ItemCheck(BottomUpTree(-1, depth))
   end
-  io.write(string.format("%d\t trees of depth %d\t check: %d\n",
+  io.write(string.format("%d trees of depth %d check: %d\n",
     iterations*2, depth, check))
 end
 
-io.write(string.format("long lived tree of depth %d\t check: %d\n",
+io.write(string.format("long lived tree of depth %d check: %d\n",
   maxdepth, ItemCheck(longlivedtree)))
 
 io.write(string.format("elapsed: %.8f\n", os.clock() - start))
diff --git a/benchmark/binary_trees.py b/benchmark/binary_trees.py
index 556278cc..16edf1de 100644
--- a/benchmark/binary_trees.py
+++ b/benchmark/binary_trees.py
@@ -18,11 +18,11 @@ def check_tree((item, left, right)):
     return item + check_tree(left) - check_tree(right)
 
 min_depth = 4
-max_depth = max(min_depth + 2, 14)
+max_depth = 12
 stretch_depth = max_depth + 1
 
 start = time.clock()
-print "stretch tree of depth %d\t check:" % stretch_depth, check_tree(make_tree(0, stretch_depth))
+print "stretch tree of depth %d check:" % stretch_depth, check_tree(make_tree(0, stretch_depth))
 
 long_lived_tree = make_tree(0, max_depth)
 
@@ -33,8 +33,8 @@ for depth in xrange(min_depth, stretch_depth, 2):
     for i in xrange(1, iterations + 1):
         check += check_tree(make_tree(i, depth)) + check_tree(make_tree(-i, depth))
 
-    print "%d\t trees of depth %d\t check:" % (iterations * 2, depth), check
+    print "%d trees of depth %d check:" % (iterations * 2, depth), check
     iterations /= 4
 
-print "long lived tree of depth %d\t check:" % max_depth, check_tree(long_lived_tree)
+print "long lived tree of depth %d check:" % max_depth, check_tree(long_lived_tree)
 print("elapsed: " + str(time.clock() - start))
\ No newline at end of file
diff --git a/benchmark/binary_trees.rb b/benchmark/binary_trees.rb
index e6e41bb3..658440c7 100644
--- a/benchmark/binary_trees.rb
+++ b/benchmark/binary_trees.rb
@@ -17,7 +17,7 @@ def bottom_up_tree(item, depth)
   [bottom_up_tree(item_item - 1, depth), item, bottom_up_tree(item_item, depth)]
 end
 
-max_depth = 14
+max_depth = 12
 min_depth = 4
 
 max_depth = min_depth + 2 if min_depth + 2 > max_depth
@@ -26,7 +26,7 @@ stretch_depth = max_depth + 1
 stretch_tree = bottom_up_tree(0, stretch_depth)
 
 start = Time.now
-puts "stretch tree of depth #{stretch_depth}\t check: #{item_check(*stretch_tree)}"
+puts "stretch tree of depth #{stretch_depth} check: #{item_check(*stretch_tree)}"
 stretch_tree = nil
 
 long_lived_tree = bottom_up_tree(0, max_depth)
@@ -44,8 +44,8 @@ min_depth.step(max_depth + 1, 2) do |depth|
     check += item_check(*temp_tree)
   end
 
-  puts "#{iterations * 2}\t trees of depth #{depth}\t check: #{check}"
+  puts "#{iterations * 2} trees of depth #{depth} check: #{check}"
 end
 
-puts "long lived tree of depth #{max_depth}\t check: #{item_check(*long_lived_tree)}"
+puts "long lived tree of depth #{max_depth} check: #{item_check(*long_lived_tree)}"
 puts "elapsed: " + (Time.now - start).to_s
diff --git a/benchmark/binary_trees.wren b/benchmark/binary_trees.wren
index c1c8a00b..eb0ee4d2 100644
--- a/benchmark/binary_trees.wren
+++ b/benchmark/binary_trees.wren
@@ -21,12 +21,12 @@ class Tree {
 }
 
 var minDepth = 4
-var maxDepth = 14
+var maxDepth = 12
 var stretchDepth = maxDepth + 1
 
 var start = OS.clock
 
-io.write("stretch tree of depth " + stretchDepth.toString + "\t check: " +
+io.write("stretch tree of depth " + stretchDepth.toString + " check: " +
     Tree.new(0, stretchDepth).check.toString)
 
 var longLivedTree = Tree.new(0, maxDepth)
@@ -48,13 +48,13 @@ while (depth < stretchDepth) {
     i = i + 1
   }
 
-  io.write((iterations * 2).toString + "\t trees of depth " + depth.toString +
-      "\t check: " + check.toString)
+  io.write((iterations * 2).toString + " trees of depth " + depth.toString +
+      " check: " + check.toString)
   iterations = iterations / 4
   depth = depth + 2
 }
 
-io.write("long lived tree of depth " + maxDepth.toString + "\t check: " +
+io.write("long lived tree of depth " + maxDepth.toString + " check: " +
     longLivedTree.check.toString)
 
 io.write("elapsed: " + (OS.clock - start).toString)
diff --git a/benchmark/fib.js b/benchmark/fib.js
deleted file mode 100644
index 9aac24f8..00000000
--- a/benchmark/fib.js
+++ /dev/null
@@ -1,13 +0,0 @@
-function fib(n) {
-  if (n < 2) return n;
-  return fib(n - 1) + fib(n - 2);
-}
-
-var start = process.hrtime();
-var i = 0;
-for (var i = 0; i < 5; i++) {
-  console.log(fib(30));
-}
-var elapsed = process.hrtime(start);
-elapsed = elapsed[0] + elapsed[1] / 1000000000;
-console.log("elapsed: " + elapsed);
diff --git a/benchmark/fib.lua b/benchmark/fib.lua
index 17e17302..c277da88 100644
--- a/benchmark/fib.lua
+++ b/benchmark/fib.lua
@@ -5,6 +5,6 @@ end
 
 local start = os.clock()
 for i = 1, 5 do
-  io.write(fib(30) .. "\n")
+  io.write(fib(28) .. "\n")
 end
 io.write(string.format("elapsed: %.8f\n", os.clock() - start))
diff --git a/benchmark/fib.py b/benchmark/fib.py
index e58ddb06..8324335a 100644
--- a/benchmark/fib.py
+++ b/benchmark/fib.py
@@ -6,5 +6,5 @@ def fib(n):
 
 start = time.clock()
 for i in range(0, 5):
-  print(fib(30))
+  print(fib(28))
 print("elapsed: " + str(time.clock() - start))
\ No newline at end of file
diff --git a/benchmark/fib.rb b/benchmark/fib.rb
index 5f224ea0..66e2bbb6 100644
--- a/benchmark/fib.rb
+++ b/benchmark/fib.rb
@@ -8,6 +8,6 @@ end
 
 start = Time.now
 for i in 0...5
-  puts fib(30)
+  puts fib(28)
 end
 puts "elapsed: " + (Time.now - start).to_s
diff --git a/benchmark/fib.wren b/benchmark/fib.wren
index a4163732..06de1bba 100644
--- a/benchmark/fib.wren
+++ b/benchmark/fib.wren
@@ -9,7 +9,7 @@ var fib = fn(n) {
 var start = OS.clock
 var i = 0
 while (i < 5) {
-  io.write(fib.call(30))
+  io.write(fib.call(28))
   i = i + 1
 }
 io.write("elapsed: " + (OS.clock - start).toString)
diff --git a/benchmark/method_call.js b/benchmark/method_call.js
deleted file mode 100644
index d7a4a8d7..00000000
--- a/benchmark/method_call.js
+++ /dev/null
@@ -1,58 +0,0 @@
-// The Great Computer Language Shootout
-// http://shootout.alioth.debian.org/
-//
-// contributed by David Hedbor
-// modified by Sjoerd Visscher
-
-
-function Toggle(start_state) {
-   this.state = start_state;
-}
-
-Toggle.prototype.value = function() {
-   return this.state;
-}
-
-Toggle.prototype.activate = function() {
-   this.state = !this.state;
-   return this;
-}
-
-
-function NthToggle (start_state, max_counter) {
-   Toggle.call(this, start_state);
-   this.count_max = max_counter;
-   this.count = 0;
-}
-
-NthToggle.prototype = new Toggle;
-
-NthToggle.prototype.activate = function() {
-   if (++this.count >= this.count_max) {
-     this.state = !this.state;
-     this.count = 0;
-   }
-   return this;
-}
-
-var start = process.hrtime();
-
-var n = 1000000;
-var i;
-var val = true;
-var toggle = new Toggle(val);
-for (i=0; i<n; i++) {
-  val = toggle.activate().value();
-}
-console.log(toggle.value() ? "true" : "false");
-
-val = true;
-var ntoggle = new NthToggle(val, 3);
-for (i=0; i<n; i++) {
-  val = ntoggle.activate().value();
-}
-console.log(ntoggle.value() ? "true" : "false");
-
-var elapsed = process.hrtime(start);
-elapsed = elapsed[0] + elapsed[1] / 1000000000;
-console.log("elapsed: " + elapsed);
diff --git a/benchmark/run_bench b/benchmark/run_bench
index d7d5989b..9b3c44ca 100755
--- a/benchmark/run_bench
+++ b/benchmark/run_bench
@@ -7,42 +7,40 @@ import re
 import subprocess
 import sys
 
+# How many times to run a given benchmark. Should be an odd number to get the
+# right median.
+NUM_TRIALS = 7
+
 BENCHMARKS = []
 
 def BENCHMARK(name, pattern):
   regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE)
   BENCHMARKS.append([name, regex, None])
 
-BENCHMARK("binary_trees", """stretch tree of depth 15\t check: -1
-32768\t trees of depth 4\t check: -32768
-8192\t trees of depth 6\t check: -8192
-2048\t trees of depth 8\t check: -2048
-512\t trees of depth 10\t check: -512
-128\t trees of depth 12\t check: -128
-32\t trees of depth 14\t check: -32
-long lived tree of depth 14\t check: -1""")
+BENCHMARK("binary_trees", """stretch tree of depth 13 check: -1
+8192 trees of depth 4 check: -8192
+2048 trees of depth 6 check: -2048
+512 trees of depth 8 check: -512
+128 trees of depth 10 check: -128
+32 trees of depth 12 check: -32
+long lived tree of depth 12 check: -1""")
 
-BENCHMARK("fib", r"""832040
-832040
-832040
-832040
-832040""")
+BENCHMARK("fib", r"""317811
+317811
+317811
+317811
+317811""")
 
 BENCHMARK("method_call", r"""true
 false""")
 
 LANGUAGES = [
   ("wren",   "../build/Release/wren", ".wren"),
-  ("js",     "node",                  ".js"),
   ("lua",    "lua",                   ".lua"),
   ("python", "python",                ".py"),
   ("ruby",   "ruby",                  ".rb")
 ]
 
-# How many times to run a given benchmark. Should be an odd number to get the
-# right median.
-NUM_TRIALS = 7
-
 results = []
 
 def green(text):
@@ -62,16 +60,17 @@ def yellow(text):
 
 
 def calc_stats(nums):
-  """Calculates the mean, median, and std deviation of a list of numbers."""
+  """Calculates the best, mean, and median of a list of numbers."""
   mean = sum(nums) / len(nums)
   nums.sort()
   median = nums[(len(nums) - 1) / 2]
   diffs = ((n - mean) * (n - mean) for n in nums)
   std_dev = math.sqrt(sum(diffs) / len(nums))
-  return [mean, median, std_dev]
+  return [nums[0], mean, median, std_dev]
 
 
-def run_benchmark_once(benchmark, language):
+def run_trial(benchmark, language):
+  """Runs one benchmark one time for one language."""
   args = [language[1], benchmark[0] + language[2]]
   out = subprocess.check_output(args, universal_newlines=True)
   match = benchmark[1].match(out)
@@ -84,6 +83,7 @@ def run_benchmark_once(benchmark, language):
 
 
 def run_benchmark_language(benchmark, language):
+  """Runs one benchmark for a number of trials for one language."""
   name = "{0} - {1}".format(benchmark[0], language[0])
   print "{0:22s}".format(name),
 
@@ -93,7 +93,7 @@ def run_benchmark_language(benchmark, language):
 
   times = []
   for i in range(0, NUM_TRIALS):
-    time = run_benchmark_once(benchmark, language)
+    time = run_trial(benchmark, language)
     if not time:
       return
     times.append(time)
@@ -105,7 +105,7 @@ def run_benchmark_language(benchmark, language):
   comparison = ""
   if language[0] == "wren":
     if benchmark[2] != None:
-      ratio = 100 * stats[1] / benchmark[2]
+      ratio = 100 * stats[0] / benchmark[2]
       comparison =  "{0:.2f}% of baseline".format(ratio)
       if ratio > 105:
         comparison = red(comparison)
@@ -123,14 +123,15 @@ def run_benchmark_language(benchmark, language):
     if ratio > 1:
       comparison = green(comparison)
 
-  print " mean: {0:.2f} median: {1:.2f} std_dev: {2:.2f} {3:s}".format(
+  print " best: {0:.2f} mean: {1:.2f} median: {2:.2f} {3:s}".format(
       stats[0], stats[1], stats[2], comparison)
 
-  results.append([name, times, stats[1]])
+  results.append([name, times, stats[0]])
   return stats
 
 
 def run_benchmark(benchmark, languages):
+  """Runs one benchmark for the given languages (or all of them)."""
   for language in LANGUAGES:
     if not languages or language[0] in languages:
       run_benchmark_language(benchmark, language)
@@ -138,6 +139,37 @@ def run_benchmark(benchmark, languages):
   del results[0:len(results)]
 
 
+# TODO(bob): Hook this up so it can be called.
+def solo_benchmark(benchmark, language):
+  """Runs a single language benchmark repeatedly, graphing the results."""
+  base = benchmark[2]
+  total = 0
+  for i in range(0, NUM_TRIALS):
+    time = run_trial(benchmark, language)
+    total += time
+    ratio = 100 * time / base
+
+    # TODO(bob): Show scale.
+
+    line = [" "] * 51
+    line[25] = "|"
+    index = 25 + int((time - base) * 200)
+    if index < 0: index = 0
+    if index > 50: index = 50
+    line[index] = "*"
+
+    comparison = "{0:.4f} ({1:6.2f}%) {2}".format(time, ratio, "".join(line))
+    if ratio > 105:
+      comparison = red(comparison)
+    if ratio < 95:
+      comparison = green(comparison)
+    print comparison
+
+  total /= NUM_TRIALS
+  print "----"
+  print "{0:.4f} ({1:6.2f}%)".format(total, 100 * total / base)
+
+
 def graph_results():
   print
 
@@ -168,10 +200,10 @@ def read_baseline():
   if os.path.exists("baseline.txt"):
     with open("baseline.txt") as f:
       for line in f.readlines():
-        name, mean, median = line.split(",")
+        name, best = line.split(",")
         for benchmark in BENCHMARKS:
           if benchmark[0] == name:
-            benchmark[2] = float(median)
+            benchmark[2] = float(best)
 
 
 def generate_baseline():
@@ -179,8 +211,7 @@ def generate_baseline():
   baseline_text = ""
   for benchmark in BENCHMARKS:
     stats = run_benchmark_language(benchmark, LANGUAGES[0])
-    baseline_text += ("{},{},{}\n".format(
-          benchmark[0], stats[0], stats[1]))
+    baseline_text += ("{},{}\n".format(benchmark[0], stats[0]))
 
   # Write them to a file.
   with open("baseline.txt", 'w') as out:
@@ -207,6 +238,7 @@ def main():
 
   read_baseline()
 
+  # Run all benchmarks.
   if args.benchmark == "all":
     for benchmark in BENCHMARKS:
       run_benchmark(benchmark, args.language)