wren/benchmark/run_all

#!/usr/bin/python

import math
import re
import subprocess
import sys

FIB_OUTPUT_PATTERN = re.compile(r"""832040
832040
832040
832040
832040
elapsed: (\d+\.\d+)""", re.MULTILINE)

BENCHMARKS = [
  ("fib", FIB_OUTPUT_PATTERN)
]

LANGUAGES = [
  ("wren",   "../build/Release/wren", ".wren"),
  ("lua",    "lua",                   ".lua"),
  ("python", "python",                ".py"),
  ("ruby",   "ruby",                  ".rb"),
  ("js",     "node",                  ".js")
]

# How many times to run a given benchmark. Should be an odd number to get the
# right median.
NUM_TRIALS = 7

results = []

def calc_stats(nums):
  """Calculates the mean, median, and std deviation of a list of numbers."""
  mean = sum(nums) / len(nums)
  nums.sort()
  median = nums[(len(nums) - 1) / 2]
  diffs = ((n - mean) * (n - mean) for n in nums)
  std_dev = math.sqrt(sum(diffs) / len(nums))
  return [mean, median, std_dev]


def run_benchmark_once(benchmark, language):
  args = [language[1], benchmark[0] + language[2]]
  out = subprocess.check_output(args, universal_newlines=True)
  match = benchmark[1].match(out)
  if match:
    return float(match.group(1))
  else:
    return None


def run_benchmark(benchmark, language):
  print "{0} - {1:10s}".format(benchmark[0], language[0]),

  times = []
  for i in range(0, NUM_TRIALS):
    times.append(run_benchmark_once(benchmark, language))
    sys.stdout.write(".")

  if None in times:
    print "error"
    return

  times.sort()
  stats = calc_stats(times)
  print " mean: {0:.4f} median: {1:.4f} std_dev: {2:.4f}".format(
      stats[0], stats[1], stats[2])

  results.append([benchmark[0] + " - " + language[0], times])

def graph_results():
  print

  # Scale everything by the highest time.
  highest = 0
  for result in results:
    time = max(result[1])
    if time > highest: highest = time

  print "{0:15s} 0.0 {1:76.4f}".format("", highest)
  for result in results:
    line = ["-"] * 80
    for time in result[1]:
      line[int(time / highest * 79)] = "O"
    print "{0:15s} {1}".format(result[0], "".join(line))

for benchmark in BENCHMARKS:
  for language in LANGUAGES:
    run_benchmark(benchmark, language)

graph_results()