| #!/usr/bin/env python |
| |
| # Copyright 2018 the V8 project authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can |
| # be found in the LICENSE file. |
| """ |
| This script averages numbers output from another script. It is useful |
| to average over a benchmark that outputs one or more results of the form |
| <key> <number> <unit> |
| key and unit are optional, but only one number per line is processed. |
| |
| For example, if |
| $ bch --allow-natives-syntax toNumber.js |
| outputs |
| Number('undefined'): 155763 |
| (+'undefined'): 193050 Kps |
| 23736 Kps |
| then |
| $ avg.py 10 bch --allow-natives-syntax toNumber.js |
| will output |
| [10/10] (+'undefined') : avg 192,240.40 stddev 6,486.24 (185,529.00 - 206,186.00) |
| [10/10] Number('undefined') : avg 156,990.10 stddev 16,327.56 (144,718.00 - 202,840.00) Kps |
| [10/10] [default] : avg 22,885.80 stddev 1,941.80 ( 17,584.00 - 24,266.00) Kps |
| """ |
| |
| # for py2/py3 compatibility |
| from __future__ import print_function |
| |
| import argparse |
| import math |
| import re |
| import signal |
| import subprocess |
| import sys |
| |
| PARSER = argparse.ArgumentParser( |
| description="A script that averages numbers from another script's output", |
| epilog="Example:\n\tavg.py 10 bash -c \"echo A: 100; echo B 120; sleep .1\"" |
| ) |
| PARSER.add_argument( |
| 'repetitions', |
| type=int, |
| help="number of times the command should be repeated") |
| PARSER.add_argument( |
| 'command', |
| nargs=argparse.REMAINDER, |
| help="command to run (no quotes needed)") |
| PARSER.add_argument( |
| '--echo', |
| '-e', |
| action='store_true', |
| default=False, |
| help="set this flag to echo the command's output") |
| |
| ARGS = vars(PARSER.parse_args()) |
| |
| if not ARGS['command']: |
| print("No command provided.") |
| exit(1) |
| |
| |
| class FieldWidth: |
| |
| def __init__(self, points=0, key=0, average=0, stddev=0, min_width=0, max_width=0): |
| self.widths = dict(points=points, key=key, average=average, stddev=stddev, |
| min=min_width, max=max_width) |
| |
| def max_widths(self, other): |
| self.widths = {k: max(v, other.widths[k]) for k, v in self.widths.items()} |
| |
| def __getattr__(self, key): |
| return self.widths[key] |
| |
| |
| def fmtS(string, width=0): |
| return "{0:<{1}}".format(string, width) |
| |
| |
| def fmtN(num, width=0): |
| return "{0:>{1},.2f}".format(num, width) |
| |
| |
| def fmt(num): |
| return "{0:>,.2f}".format(num) |
| |
| |
| def format_line(points, key, average, stddev, min_value, max_value, |
| unit_string, widths): |
| return "{:>{}}; {:<{}}; {:>{}}; {:>{}}; {:>{}}; {:>{}}; {}".format( |
| points, widths.points, |
| key, widths.key, |
| average, widths.average, |
| stddev, widths.stddev, |
| min_value, widths.min, |
| max_value, widths.max, |
| unit_string) |
| |
| |
| def fmt_reps(msrmnt): |
| rep_string = str(ARGS['repetitions']) |
| return "[{0:>{1}}/{2}]".format(msrmnt.size(), len(rep_string), rep_string) |
| |
| |
| class Measurement: |
| |
| def __init__(self, key, unit): |
| self.key = key |
| self.unit = unit |
| self.values = [] |
| self.average = 0 |
| self.count = 0 |
| self.M2 = 0 |
| self.min = float("inf") |
| self.max = -float("inf") |
| |
| def addValue(self, value): |
| try: |
| num_value = float(value) |
| self.values.append(num_value) |
| self.min = min(self.min, num_value) |
| self.max = max(self.max, num_value) |
| self.count = self.count + 1 |
| delta = num_value - self.average |
| self.average = self.average + delta / self.count |
| delta2 = num_value - self.average |
| self.M2 = self.M2 + delta * delta2 |
| except ValueError: |
| print("Ignoring non-numeric value", value) |
| |
| def status(self, widths): |
| return "{} {}: avg {} stddev {} ({} - {}) {}".format( |
| fmt_reps(self), |
| fmtS(self.key, widths.key), fmtN(self.average, widths.average), |
| fmtN(self.stddev(), widths.stddev), fmtN(self.min, widths.min), |
| fmtN(self.max, widths.max), fmtS(self.unit_string())) |
| |
| def result(self, widths): |
| return format_line(self.size(), self.key, fmt(self.average), |
| fmt(self.stddev()), fmt(self.min), |
| fmt(self.max), self.unit_string(), |
| widths) |
| |
| def unit_string(self): |
| if not self.unit: |
| return "" |
| return self.unit |
| |
| def variance(self): |
| if self.count < 2: |
| return float('NaN') |
| return self.M2 / (self.count - 1) |
| |
| def stddev(self): |
| return math.sqrt(self.variance()) |
| |
| def size(self): |
| return len(self.values) |
| |
| def widths(self): |
| return FieldWidth( |
| points=len("{}".format(self.size())) + 2, |
| key=len(self.key), |
| average=len(fmt(self.average)), |
| stddev=len(fmt(self.stddev())), |
| min_width=len(fmt(self.min)), |
| max_width=len(fmt(self.max))) |
| |
| |
| def result_header(widths): |
| return format_line("#/{}".format(ARGS['repetitions']), |
| "id", "avg", "stddev", "min", "max", "unit", widths) |
| |
| |
| class Measurements: |
| |
| def __init__(self): |
| self.all = {} |
| self.default_key = '[default]' |
| self.max_widths = FieldWidth( |
| points=len("{}".format(ARGS['repetitions'])) + 2, |
| key=len("id"), |
| average=len("avg"), |
| stddev=len("stddev"), |
| min_width=len("min"), |
| max_width=len("max")) |
| self.last_status_len = 0 |
| |
| def record(self, key, value, unit): |
| if not key: |
| key = self.default_key |
| if key not in self.all: |
| self.all[key] = Measurement(key, unit) |
| self.all[key].addValue(value) |
| self.max_widths.max_widths(self.all[key].widths()) |
| |
| def any(self): |
| if self.all: |
| return next(iter(self.all.values())) |
| return None |
| |
| def print_results(self): |
| print("{:<{}}".format("", self.last_status_len), end="\r") |
| print(result_header(self.max_widths), sep=" ") |
| for key in sorted(self.all): |
| print(self.all[key].result(self.max_widths), sep=" ") |
| |
| def print_status(self): |
| status = "No results found. Check format?" |
| measurement = MEASUREMENTS.any() |
| if measurement: |
| status = measurement.status(MEASUREMENTS.max_widths) |
| print("{:<{}}".format(status, self.last_status_len), end="\r") |
| self.last_status_len = len(status) |
| |
| |
| MEASUREMENTS = Measurements() |
| |
| |
| def signal_handler(signum, frame): |
| print("", end="\r") |
| MEASUREMENTS.print_results() |
| sys.exit(0) |
| |
| |
| signal.signal(signal.SIGINT, signal_handler) |
| |
| SCORE_REGEX = (r'\A((console.timeEnd: )?' |
| r'(?P<key>[^\s:,]+)[,:]?)?' |
| r'(^\s*|\s+)' |
| r'(?P<value>[0-9]+(.[0-9]+)?)' |
| r'\ ?(?P<unit>[^\d\W]\w*)?[.\s]*\Z') |
| |
| for x in range(0, ARGS['repetitions']): |
| proc = subprocess.Popen(ARGS['command'], stdout=subprocess.PIPE) |
| for line in proc.stdout: |
| if ARGS['echo']: |
| print(line.decode(), end="") |
| for m in re.finditer(SCORE_REGEX, line.decode()): |
| MEASUREMENTS.record(m.group('key'), m.group('value'), m.group('unit')) |
| proc.wait() |
| if proc.returncode != 0: |
| print("Child exited with status %d" % proc.returncode) |
| break |
| |
| MEASUREMENTS.print_status() |
| |
| # Print final results |
| MEASUREMENTS.print_results() |