diff --git a/configs/running-jikesrvm-marksweep-compare.yml b/configs/running-jikesrvm-marksweep-compare.yml deleted file mode 100644 index 6afe280..0000000 --- a/configs/running-jikesrvm-marksweep-compare.yml +++ /dev/null @@ -1,14 +0,0 @@ -includes: - - "./running-jikesrvm-base.yml" - -runtimes: - MarkSweep_Trunk: - type: JikesRVM - home: "./build/MarkSweep_Trunk_x86_64_m32-linux" - MarkSweep_Branch: - type: JikesRVM - home: "./build/MarkSweep_Branch_x86_64_m32-linux" - -configs: - - "MarkSweep_Trunk" - - "MarkSweep_Branch" diff --git a/configs/running-jikesrvm-nogc-compare.yml b/configs/running-jikesrvm-nogc-compare.yml deleted file mode 100644 index e827286..0000000 --- a/configs/running-jikesrvm-nogc-compare.yml +++ /dev/null @@ -1,14 +0,0 @@ -includes: - - "./running-jikesrvm-base.yml" - -runtimes: - NoGC_Trunk: - type: JikesRVM - home: "./build/NoGC_Trunk_x86_64_m32-linux" - NoGC_Branch: - type: JikesRVM - home: "./build/NoGC_Branch_x86_64_m32-linux" - -configs: - - "NoGC_Trunk|heap4g" - - "NoGC_Branch|heap4g" diff --git a/configs/running-jikesrvm-semispace-compare.yml b/configs/running-jikesrvm-semispace-compare.yml deleted file mode 100644 index 017cc73..0000000 --- a/configs/running-jikesrvm-semispace-compare.yml +++ /dev/null @@ -1,14 +0,0 @@ -includes: - - "./running-jikesrvm-base.yml" - -runtimes: - SemiSpace_Trunk: - type: JikesRVM - home: "./build/SemiSpace_Trunk_x86_64_m32-linux" - SemiSpace_Branch: - type: JikesRVM - home: "./build/SemiSpace_Branch_x86_64_m32-linux" - -configs: - - "SemiSpace_Trunk" - - "SemiSpace_Branch" diff --git a/configs/running-openjdk-concurrentimmix-compare.yml b/configs/running-openjdk-concurrentimmix-compare.yml new file mode 100644 index 0000000..661c0e3 --- /dev/null +++ b/configs/running-openjdk-concurrentimmix-compare.yml @@ -0,0 +1,6 @@ +includes: + - "./running-openjdk-base.yml" + +configs: + - "jdk-mmtk-trunk|concurrentimmix|common_mmtk" + - "jdk-mmtk-branch|concurrentimmix|common_mmtk" diff --git a/configs/running-openjdk-lisp2-compare.yml b/configs/running-openjdk-lisp2-compare.yml new file mode 100644 index 0000000..3c6f419 --- /dev/null +++ b/configs/running-openjdk-lisp2-compare.yml @@ -0,0 +1,6 @@ +includes: + - "./running-openjdk-base.yml" + +configs: + - "jdk-mmtk-trunk|lisp2|common_mmtk" + - "jdk-mmtk-branch|lisp2|common_mmtk" diff --git a/configs/running-openjdk-ovc-compare.yml b/configs/running-openjdk-ovc-compare.yml new file mode 100644 index 0000000..860d2e4 --- /dev/null +++ b/configs/running-openjdk-ovc-compare.yml @@ -0,0 +1,6 @@ +includes: + - "./running-openjdk-base.yml" + +configs: + - "jdk-mmtk-trunk|ovc|common_mmtk" + - "jdk-mmtk-branch|ovc|common_mmtk" diff --git a/scripts/common.sh b/scripts/common.sh index 966eadd..b7524be 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -373,7 +373,27 @@ commit_result_repo() { git add . git commit -m "$message" git pull --rebase # pull any new commit (if any) - git push + git push + else + echo "SKIP_UPLOAD_RESULT is set, skip uploading result" + fi +} + +# commit_result_repo_at 'dest_dir' 'message' +# Like commit_result_repo, but commits/pushes an explicit dest_dir (as set up +# by checkout_result_repo_at) instead of the single global $result_repo_dir - +# for callers that check out more than one branch of the result repo at once. +commit_result_repo_at() { + dest_dir=$1 + message=$2 + + if [[ -z $SKIP_UPLOAD_RESULT ]]; then + cd $dest_dir + + git add . + git commit -m "$message" + git pull --rebase # pull any new commit (if any) + git push else echo "SKIP_UPLOAD_RESULT is set, skip uploading result" fi diff --git a/scripts/compare_plot.py b/scripts/compare_plot.py new file mode 100644 index 0000000..bd569c0 --- /dev/null +++ b/scripts/compare_plot.py @@ -0,0 +1,349 @@ +""" Compares two builds (e.g. trunk vs a PR branch) that were run together in + one running-ng invocation (see running-openjdk-*-compare.yml), producing: + - /report.md: a "Time" Markdown table (one column per + Total/Mutator/STW) and, only for plans opted into `latency: true` in + (see configs/openjdk-plot.yml), a "Tail latency" table + (one column per p50/p90/p99/p99.9/p99.99) - one row per benchmark + plus trailing min/max/mean/geomean summary rows, each cell branch + normalized to trunk as a percentage difference with a 95% + confidence interval. + - /plots/_time.png: the same Time grouping as a + chart - 3 bars per benchmark. + - /plots/_latency.png: the same Tail latency + grouping as a chart - 5 bars per benchmark. + - /plots/_.png: one chart per any other + metric the logs happen to contain (an MMTk stat outside the two + groups above) - a single bar per benchmark. + Every bar is build2 (branch)'s value normalized to build1 (trunk)'s mean + for that benchmark - trunk itself isn't plotted, since normalized to + itself it's always ~1.0 and adds nothing. Bars have 95% + confidence-interval error bars, and each chart has trailing + min/max/mean/geomean summary bars (one per metric in the chart). + + Usage: python compare_plot.py + build1 is the normalization baseline (trunk); build2 (branch) is plotted + relative to it. is a configs/*-plot.yml path (e.g. + configs/openjdk-plot.yml) - only used to look up whether is + latency-enabled. +""" + +import sys +import os +import numpy as np +from scipy import stats +from scipy.stats import gmean + +import parse +import plot as plotmod + +if len(sys.argv) != 8: + print("Usage: python compare_plot.py ") + sys.exit(1) + +log_dir = sys.argv[1] +plan = sys.argv[2] +build1 = sys.argv[3] +build2 = sys.argv[4] +invocations = int(sys.argv[5]) +output_dir = sys.argv[6] +plot_config_path = sys.argv[7] + +run_id, results, commit_info = parse.parse_run(log_dir, invocations) +results = [r for r in results if r is not None] + +benchmarks = sorted(set(r['benchmark'] for r in results)) + +RESERVED_KEYS = {'log_name', 'benchmark', 'build', 'status', 'succeeded_runs'} +LATENCY_PREFIX = 'latency.' + +plot_config = parse.parse_yaml(plot_config_path) +plan_config = parse.get_config_for_plan(plot_config, plan) +LATENCY_ENABLED = bool(plan_config) and plan_config.get('latency', False) +if not LATENCY_ENABLED: + print("Plan %s is not configured for latency (see %s) - skipping latency metrics." % (plan, plot_config_path)) + +CONFIDENCE = 0.95 + +# The only metrics we plot/tabulate: the 3 time metrics merged into a single +# chart/table (3 bars/columns per benchmark), and, for latency-enabled plans, +# the 5 latency percentiles likewise merged into one chart/table. Colors +# reused from plot.py's own SECONDARY_METRICS/LATENCY_SECONDARY_METRICS so +# this reads as part of the same palette family as the history reports. +TIME_METRICS = [ + ('time.total', 'Total', plotmod.INK_PRIMARY), + ('time.other', 'Mutator', "#2a78d6"), + ('time.stw', 'STW', "#4a3aa7"), +] +LATENCY_METRICS = [ + ('latency.p50', 'p50', "#b9b6ac"), + ('latency.p90', 'p90', "#9a978e"), + ('latency.p99', 'p99', "#7b7972"), + ('latency.p999', 'p99.9', "#5c5a54"), + ('latency.p9999', 'p99.99', plotmod.BASELINE_COLOR), +] + + +def discovered_metric_keys(): + """ Every list-valued metric key present across any parsed result (union, + since not every benchmark/build necessarily has the same set of MMTk + stat keys) - used to check which of TIME_METRICS/LATENCY_METRICS + actually have data. Latency percentiles are only included if the + plan is latency-enabled (see LATENCY_ENABLED above) - like + history_report.py, we don't plot them for plans nobody asked to + track latency for, even though the underlying benchmarks + (cassandra/h2/lusearch/tomcat) print tail latency regardless of + which plan is running. """ + keys = set() + for r in results: + for k, v in r.items(): + if k in RESERVED_KEYS: + continue + if k.startswith(LATENCY_PREFIX) and not LATENCY_ENABLED: + continue + if isinstance(v, list): + keys.add(k) + return keys + + +def get_result(build, benchmark): + matches = [r for r in results if r['benchmark'] == benchmark and r['build'] == build] + return matches[0] if matches else None + + +def mean_ci(arr, confidence=CONFIDENCE): + """ Mean and 95% t-distribution confidence-interval half-width - same + formula the old compare_report.py used. """ + arr = np.asarray(arr, dtype=float) + mean = np.mean(arr) + if len(arr) > 1: + ci = stats.sem(arr) * stats.t.ppf((1 + confidence) / 2., len(arr) - 1) + else: + ci = 0.0 + return mean, ci + + +# --------------------------------------------------------------------------- +# report.md: one Markdown table per metric group (Time, Tail latency) - one +# column per metric in the group, one row per benchmark. Uses the same +# compute_branch_ratios as the plots (see below), so the tables and the +# charts always agree. +# --------------------------------------------------------------------------- + +def format_pct_ci(mean, ci): + return '%+.2f%% ±%.2f%%' % ((mean - 1) * 100, ci * 100) + + +def build_metric_table(metrics): + """ metrics: list of (metric_key, label, color) tuples, e.g. TIME_METRICS. + Returns a Markdown table with one column per metric that actually + has data, or None if none of them do. """ + ratios_by_metric = {} + for key, _, _ in metrics: + ratios = compute_branch_ratios(key) + if len(ratios) > 0: + ratios_by_metric[key] = ratios + + if len(ratios_by_metric) == 0: + return None + + present_metrics = [(key, label) for key, label, _ in metrics if key in ratios_by_metric] + ordered_benchmarks = [bm for bm in benchmarks if any(bm in r for r in ratios_by_metric.values())] + + lines = [] + lines.append('|Benchmark|' + '|'.join(label for _, label in present_metrics) + '|') + lines.append('|:-------:|' + '|'.join([':-------------------:'] * len(present_metrics)) + '|') + + for bm in ordered_benchmarks: + cells = [] + for key, _ in present_metrics: + ratios = ratios_by_metric[key] + cells.append(format_pct_ci(*ratios[bm]) if bm in ratios else ':x:') + lines.append('|%s|%s|' % (bm, '|'.join(cells))) + + # Trailing min/max/mean/geomean summary rows, matching the summary bars + # on the corresponding chart - no CI here, same as those bars. + for summary_label in SUMMARY_LABELS: + cells = [] + for key, _ in present_metrics: + summary_mean = summary_stats(ratios_by_metric[key])[summary_label] + cells.append('%+.2f%%' % ((summary_mean - 1) * 100)) + lines.append('|**%s**|%s|' % (summary_label, '|'.join(cells))) + + return '\n'.join(lines) + + +def build_report(): + lines = [] + lines.append('%s (%s)' % (plan, run_id)) + lines.append('') + + time_table = build_metric_table(TIME_METRICS) + if time_table: + lines.append('### Time') + lines.append('') + lines.append(time_table) + lines.append('') + + if LATENCY_ENABLED: + latency_table = build_metric_table(LATENCY_METRICS) + if latency_table: + lines.append('### Tail latency') + lines.append('') + lines.append(latency_table) + lines.append('') + + return '\n'.join(lines) + + +# --------------------------------------------------------------------------- +# plots: one normalized, grouped bar chart per metric group (time, latency, +# or a single other metric on its own) - build2 (branch) only, since build1 +# (trunk) normalized to itself is always ~1.0 and isn't worth drawing. +# --------------------------------------------------------------------------- + +def compute_branch_ratios(metric_key): + """ For each benchmark, build2 (branch)'s values normalized to build1 + (trunk)'s mean for that benchmark. Returns an ordered dict of + benchmark -> (mean_ratio, ci), skipping benchmarks where either + build has no (or all-zero) data for this metric. """ + per_benchmark = {} + for bm in benchmarks: + trunk_result = get_result(build1, bm) + branch_result = get_result(build2, bm) + if trunk_result is None or branch_result is None: + continue + trunk_vals = trunk_result.get(metric_key, []) + branch_vals = branch_result.get(metric_key, []) + if len(trunk_vals) == 0 or len(branch_vals) == 0: + continue + trunk_mean_abs = np.mean(trunk_vals) + # Some MMTk stats use +-inf as a "not applicable to this + # benchmark/plan" sentinel (e.g. total-work.time.max/min). Treat + # that the same as a zero baseline - can't normalize to it, and + # inf/inf or x/inf would otherwise silently produce NaN, which + # breaks the whole chart's axis range further down. + if trunk_mean_abs == 0 or not np.isfinite(trunk_mean_abs): + continue + ratio_mean, ratio_ci = mean_ci(plotmod.normalize_to(branch_vals, trunk_mean_abs)) + if not np.isfinite(ratio_mean): + continue + per_benchmark[bm] = (ratio_mean, ratio_ci) + return per_benchmark + + +def summary_stats(per_benchmark_ratios): + means = [mean for mean, _ in per_benchmark_ratios.values()] + return { + 'min': min(means), + 'max': max(means), + 'mean': np.mean(means), + 'geomean': gmean(means), + } + + +SUMMARY_LABELS = ['min', 'max', 'mean', 'geomean'] + + +def plot_metric_group(metrics, chart_title, filename, plots_dir): + """ metrics: list of (metric_key, label, color) tuples, e.g. TIME_METRICS. + One bar trace per metric, sharing an x-axis of benchmark + a blank + spacer + SUMMARY_LABELS. Metrics with no data at all are dropped; + returns None (and plots nothing) if none of them have any. """ + import plotly.graph_objs as go + + ratios_by_metric = {} + for key, _, _ in metrics: + ratios = compute_branch_ratios(key) + if len(ratios) == 0: + print("Skipping %s: no benchmark has data for both builds." % key) + continue + ratios_by_metric[key] = ratios + + if len(ratios_by_metric) == 0: + return None + + ordered_benchmarks = [bm for bm in benchmarks if any(bm in r for r in ratios_by_metric.values())] + x = ordered_benchmarks + [' '] + SUMMARY_LABELS + + traces = [] + all_points = [] + for key, label, color in metrics: + if key not in ratios_by_metric: + continue + ratios = ratios_by_metric[key] + summary = summary_stats(ratios) + + y, err = [], [] + for bm in ordered_benchmarks: + if bm in ratios: + mean, ci = ratios[bm] + y.append(mean) + err.append(ci) + else: + y.append(None) + err.append(0) + y.append(None) + err.append(0) + for summary_label in SUMMARY_LABELS: + y.append(summary[summary_label]) + err.append(0) + + traces.append(go.Bar(name=label, x=x, y=y, error_y=dict(type='data', array=err, visible=True), marker_color=color)) + all_points.extend((v, e) for v, e in zip(y, err) if v is not None) + + # Bars start at 0, like a normal bar chart - just pad the top a little + # so the tallest bar/error-bar cap isn't flush against the plot edge. + y_high = max(v + e for v, e in all_points) + y_pad = max(y_high * 0.05, 0.02) + + width = max(900, 45 * len(x)) + fig = go.Figure(data=traces) + fig.update_layout( + barmode='group', + width=width, + height=550, + font={"family": plotmod.FONT_FAMILY, "color": plotmod.INK_PRIMARY}, + plot_bgcolor=plotmod.SURFACE, + paper_bgcolor=plotmod.SURFACE, + title={ + "text": "%s - %s (normalized to %s)" % (plan, chart_title, build1), + "font": {"size": 16, "color": plotmod.INK_PRIMARY}, + }, + xaxis={"tickangle": -30, "gridcolor": plotmod.GRIDLINE, "linecolor": plotmod.AXIS_LINE}, + yaxis={ + "title": chart_title, + "gridcolor": plotmod.GRIDLINE, + "linecolor": plotmod.AXIS_LINE, + "range": [0, y_high + y_pad], + }, + legend={"orientation": "h", "y": -0.3}, + ) + fig.add_hline(y=1.0, line_dash="dash", line_color=plotmod.INK_MUTED) + + out_path = os.path.join(plots_dir, filename) + fig.write_image(out_path, scale=2) + return filename + + +def main(): + os.makedirs(output_dir, exist_ok=True) + plots_dir = os.path.join(output_dir, 'plots') + os.makedirs(plots_dir, exist_ok=True) + + with open(os.path.join(output_dir, 'report.md'), 'w') as f: + f.write(build_report()) + + discovered = discovered_metric_keys() + + time_metrics = [m for m in TIME_METRICS if m[0] in discovered] + if time_metrics: + plot_metric_group(time_metrics, "Time", "%s_time.png" % plan, plots_dir) + + if LATENCY_ENABLED: + latency_metrics = [m for m in LATENCY_METRICS if m[0] in discovered] + if latency_metrics: + plot_metric_group(latency_metrics, "Tail latency", "%s_latency.png" % plan, plots_dir) + + +if __name__ == '__main__': + main() diff --git a/scripts/compare_report.py b/scripts/compare_report.py deleted file mode 100644 index 59c73ed..0000000 --- a/scripts/compare_report.py +++ /dev/null @@ -1,139 +0,0 @@ -import parse -import sys -import os - -output = '' -def append_output(msg): - global output - output += msg + '\n' - -# log folder -if len(sys.argv) != 6: - print("Usage: python compare_report.py ") - sys.exit(1) - -folder = sys.argv[1] -plan = sys.argv[2] -build1 = sys.argv[3] -build2 = sys.argv[4] -expected_invocations = int(sys.argv[5]) -run_id = os.path.basename(os.path.normpath(folder)) - -# list all the logs -logs = parse.list_logs(folder) - -results = [] -for l in logs: - results.append(parse.parse_log(os.path.join(folder, l), expected_invocations)) - -# benchmarks -benchmarks = [r['benchmark'] for r in results] -benchmarks = list(set(benchmarks)) -benchmarks.sort() - -append_output('%s (%s)' % (plan, run_id)) -append_output('-----') -append_output('|Benchmark| Trunk(ms) | | | Branch(ms) | | | Diff | |') -append_output('|:-------:|:----------:|:---------------------:|:------:|:-----------:|:---------------------:|:------:|:----:|:---------------------:|') -append_output('| | mean | mean without outliers | median | mean | mean without outliers | median | mean | mean without outliers |') - -# for each benchmark -for bm in benchmarks: - def average_execution_time(r): - if len(r['execution_times']) == 0: - return None - else: - return sum(r['execution_times']) / len(r['execution_times']) - - def get_result(build): - result = None - - has_result = [r for r in results if r['benchmark'] == bm and r['build'] == build][:1] - if len(has_result) != 0: - result = has_result[0] - - return result - - def get_statistics(result): - ret = {} - - import numpy as np - from scipy import stats - arr = np.array(result['execution_times']) - - confidence = 0.95 - - # mean - ret['mean'] = np.mean(arr) - # confidence interval - ret['mean_ci'] = stats.sem(arr) * stats.t.ppf((1 + confidence) / 2., len(arr) - 1) - # and median - ret['median'] = np.median(arr) - - # outliers - z_score = stats.zscore(arr) - # only keep data whose zscore is within 3 - filtered = [r for r, z in zip(arr, z_score) if z < 3] - # number of outliers and mean without outliers - ret['n_outliers'] = len(arr) - len(filtered) - ret['mean_without_outliers'] = np.mean(filtered) - ret['mean_without_outliers_ci'] = stats.sem(filtered) * stats.t.ppf((1 + confidence) / 2., len(filtered) - 1) - - return ret - - trunk_result = get_result(build1) - branch_result = get_result(build2) - - def format_build_statistics(result): - s = get_statistics(result) - - text_mean = None - - if result['status'] == 'success': - text_mean = '%.2f ±%.2f' % (s['mean'], s['mean_ci']) - elif result['status'] == 'partial_fail': - text_mean = '%.2f ±%.2f :warning: %d/%d failed' % (s['mean'], s['mean_ci'], expected_invocations - result['succeeded_runs'], expected_invocations) - elif result['status'] == 'fail': - text_mean = ':x:' - - text_mean_without_outliers = None - if result['status'] == 'fail': - text_mean_without_outliers = ':x:' - else: - if s['n_outliers'] != 0: - text_mean_without_outliers = '%.2f ±%.2f :warning: %d removed' % (s['mean_without_outliers'], s['mean_without_outliers_ci'], s['n_outliers']) - else: - text_mean_without_outliers = '%.2f ±%.2f' % (s['mean_without_outliers'], s['mean_without_outliers_ci']) - - text_median = None - if result['status'] != 'fail': - text_median = s['median'] - else: - text_median = ':x:' - - return text_mean, text_mean_without_outliers, text_median, s - - def format_diff(stats1, stats2, key, should_highlight): - # mean diff - diff = 0 - if stats1[key] is not None and stats2[key] is not None: - diff = (stats2[key] - stats1[key]) / stats1[key] - diff_text = '%+.2f%%' % (diff * 100) - - if should_highlight: - if diff >= 0.01: - diff_text += ' :red_square:' - elif diff <= -0.01: - diff_text += ' :green_square:' - return diff_text - - trunk_mean, trunk_mean_without_outliers, trunk_median, trunk_stats = format_build_statistics(trunk_result) - branch_mean, branch_mean_without_outliers, branch_median, branch_stats = format_build_statistics(branch_result) - - mean_diff = format_diff(trunk_stats, branch_stats, 'mean', False) - mean_without_outliers_diff = format_diff(trunk_stats, branch_stats, 'mean_without_outliers', True) - - append_output('|%s|%s|%s|%s|%s|%s|%s|%s|%s|' % (bm, trunk_mean, trunk_mean_without_outliers, trunk_median, branch_mean, branch_mean_without_outliers, branch_median, mean_diff, mean_without_outliers_diff)) - -append_output('') -print(output) diff --git a/scripts/jikesrvm-compare.sh b/scripts/jikesrvm-compare.sh deleted file mode 100755 index f132c23..0000000 --- a/scripts/jikesrvm-compare.sh +++ /dev/null @@ -1,78 +0,0 @@ -set -ex - -# include common.sh -. $(dirname "$0")/common.sh - -jikesrvm_binding_trunk=$(realpath $1) -mmtk_core_trunk=$(realpath $2) -jikesrvm_binding_branch=$(realpath $3) -mmtk_core_branch=$(realpath $4) -output_file=$(realpath -m $5) - -jikesrvm_trunk_rev=$(git -C $jikesrvm_binding_trunk rev-parse HEAD) -mmtk_trunk_rev=$(git -C $mmtk_core_trunk rev-parse HEAD) -jikesrvm_branch_rev=$(git -C $jikesrvm_binding_branch rev-parse HEAD) -mmtk_branch_rev=$(git -C $mmtk_core_branch rev-parse HEAD) - -# JikesRVM root -jikesrvm_trunk=$jikesrvm_binding_trunk/repos/jikesrvm -jikesrvm_branch=$jikesrvm_binding_branch/repos/jikesrvm - -# Edit jikesrvm binding Cargo.toml to use local path for mmtk core - note: this makes this script not repeatable -jikesrvm_binding_use_local_mmtk $jikesrvm_binding_trunk -if [ "$jikesrvm_binding_branch" != "$jikesrvm_binding_trunk" ]; then - jikesrvm_binding_use_local_mmtk $jikesrvm_binding_branch -fi - -# Build - JikesRVM buildit script requires current dir to be JikesRVM root dir -ensure_empty_dir $kit_build -ensure_empty_dir $kit_upload -ensure_empty_dir $log_dir - -# Trunk -rm -rf $jikesrvm_binding_trunk/repos/mmtk-core -ln -sfn $mmtk_core_trunk $jikesrvm_binding_trunk/repos/mmtk-core - -# Branch -rm -rf $jikesrvm_binding_branch/repos/mmtk-core -ln -sfn $mmtk_core_branch $jikesrvm_binding_branch/repos/mmtk-core - -# Run -cd $kit_root - -echo "JikesRVM" >> $output_file -echo "====" >> $output_file - -echo "* binding_trunk: [$jikesrvm_trunk_rev](https://github.com/mmtk/mmtk-jikesrvm/commit/$jikesrvm_trunk_rev)" >> $output_file -echo "* trunk: [$mmtk_trunk_rev](https://github.com/mmtk/mmtk-core/commit/$mmtk_trunk_rev)" >> $output_file -echo "* binding_branch: [$jikesrvm_branch_rev](https://github.com/mmtk/mmtk-jikesrvm/commit/$jikesrvm_branch_rev)" >> $output_file -echo "* branch: [$mmtk_branch_rev](https://github.com/mmtk/mmtk-core/commit/$mmtk_branch_rev)" >> $output_file - -echo "" >> $output_file - -# Python venv -start_venv python-env -pip3 install -r scripts/requirements.txt - -run_exp() { - build_config=$1 - plan=$2 - run_config=$3 - heap_modifier=$4 - - cd $jikesrvm_trunk - build_jikesrvm_with_mmtk $jikesrvm_binding_trunk $build_config $plan"_Trunk_x86_64_m32-linux" - - cd $jikesrvm_branch - build_jikesrvm_with_mmtk $jikesrvm_binding_branch $build_config $plan"_Branch_x86_64_m32-linux" - - run_id=$(run_benchmarks $log_dir $run_config $heap_modifier $compare_invocations) - python $kit_root/scripts/compare_report.py $log_dir/$run_id $plan $plan"_Trunk" $plan"_Branch" $compare_invocations >> $output_file -} - -# NoGC -run_exp RFastAdaptiveNoGC NoGC $kit_root/configs/running-jikesrvm-nogc-compare.yml 0 -# SemiSpace -run_exp RFastAdaptiveSemiSpace SemiSpace $kit_root/configs/running-jikesrvm-semispace-compare.yml 6 -# MarkSweep -run_exp RFastAdaptiveMarkSweep MarkSweep $kit_root/configs/running-jikesrvm-marksweep-compare.yml 6 diff --git a/scripts/openjdk-build.sh b/scripts/openjdk-build.sh index 84aba3d..16b3781 100755 --- a/scripts/openjdk-build.sh +++ b/scripts/openjdk-build.sh @@ -15,6 +15,14 @@ if [ -n "$2" ]; then else mmtk_core= fi +# build_path: optional name for the build dir under $kit_build/ (and +# $kit_upload/). Defaults to 'jdk-mmtk' (the name openjdk-run-plan.sh and +# every running-openjdk-*-complete.yml config expect). Callers that need +# more than one build to coexist under the same $kit_build - e.g. a +# trunk/branch pair for PR comparison, restored from two separate caches - +# pass distinct names here (see running-openjdk-*-compare.yml, which +# reference 'jdk-mmtk-trunk'/'jdk-mmtk-branch' explicitly). +build_path=${3:-jdk-mmtk} ensure_empty_dir $kit_build ensure_empty_dir $kit_upload @@ -23,10 +31,10 @@ ensure_empty_dir $kit_upload build_probes build_openjdk_probe -# Build OpenJDK+MMTk. The result is left at $kit_build/jdk-mmtk (and -# bundled at $kit_upload/jdk-mmtk) for openjdk-run-plan.sh to run against. -build_openjdk_with_mmtk $openjdk_binding release jdk-mmtk +# Build OpenJDK+MMTk. The result is left at $kit_build/$build_path (and +# bundled at $kit_upload/$build_path) for openjdk-run-plan.sh to run against. +build_openjdk_with_mmtk $openjdk_binding release $build_path # Record which commits went into this build, so openjdk-run-plan.sh can # carry it into each run's log folder. -write_commit_info $kit_build/jdk-mmtk/commit-info.yml mmtk-openjdk $openjdk_binding $mmtk_core +write_commit_info $kit_build/$build_path/commit-info.yml mmtk-openjdk $openjdk_binding $mmtk_core diff --git a/scripts/openjdk-compare-run-plan.sh b/scripts/openjdk-compare-run-plan.sh new file mode 100644 index 0000000..1258716 --- /dev/null +++ b/scripts/openjdk-compare-run-plan.sh @@ -0,0 +1,86 @@ +set -ex + +# include common.sh +. $(dirname "$0")/common.sh + +# Run a single plan's trunk-vs-branch OpenJDK comparison, generate a +# comparison table and per-metric plots (see compare_plot.py), then save the +# result. Assumes both runtimes referenced by $config already exist under +# $kit_build - i.e. openjdk-build.sh has already been run twice, once with +# build_path=jdk-mmtk-trunk and once with build_path=jdk-mmtk-branch (see +# running-openjdk-*-compare.yml, which reference exactly those two runtime +# names). +# +# The full result (raw logs + report.md + plots) is pushed to +# $RESULT_REPO_BRANCH (private). The plot images are additionally pushed to +# gh-pages (public), which is the only copy a PR comment can actually link +# to - see the note further down. + +# plan_name: result dir name under result_repo/openjdk/pr-/ (e.g. stickyimmix) +plan_name=$1 +# config: running-ng config filename under configs/ (e.g. running-openjdk-stickyimmix-compare.yml) +config=$2 +# heap_modifier: passed straight to run_benchmarks +heap_modifier=$3 +# pr_number: the mmtk-core pull request this comparison is for +pr_number=$4 + +ensure_empty_dir $log_dir +checkout_result_repo + +# Run - the compare config runs both jdk-mmtk-trunk and jdk-mmtk-branch as +# part of the same running-ng invocation, so this produces one run_id +# containing both sides' logs together. +run_id=$(run_benchmarks $log_dir $kit_root/configs/$config $heap_modifier $compare_invocations) + +# Carry both sides' commit info into this run's log folder, so it's clear +# which mmtk-core/mmtk-openjdk commits were compared, without needing to +# cross-reference the workflow run that produced it. +for side in trunk branch; do + commit_info=$kit_build/jdk-mmtk-$side/commit-info.yml + if [ -f "$commit_info" ]; then + cp $commit_info $log_dir/$run_id/commit-info-$side.yml + fi +done + +# Generate the comparison table (report.md) and one normalized bar chart per +# metric (plots/*.png), written straight into this run's own log folder so +# they get pushed to the result repo alongside the raw logs by the copy +# below - no separate publishing step needed. +start_venv python-env +pip3 install -r scripts/requirements.txt +python $kit_root/scripts/compare_plot.py $log_dir/$run_id $plan_name jdk-mmtk-trunk jdk-mmtk-branch $compare_invocations $log_dir/$run_id $kit_root/configs/openjdk-plot.yml +leave_venv + +# Copy the report out to a fixed path and record the run id in its own file, +# so the calling workflow can pick both up regardless of run_id's value +# without having to parse this script's own stdout (which also carries the +# venv/pip/set-x noise above). +cp $log_dir/$run_id/report.md $kit_root/report.md +echo $run_id > $kit_root/run_id.txt + +# Save result. The path embeds the PR number and this run's id, so results +# from repeated/different runs of the same plan on the same PR don't +# collide or overwrite each other. +RESULT_DIR=$result_repo_dir/openjdk/pr-$pr_number/$plan_name +mkdir -p $RESULT_DIR +cp -r $log_dir/$run_id $RESULT_DIR + +# Commit result. +commit_result_repo 'OpenJDK compare PR #'$pr_number' ('$plan_name'): '$run_id + +# ci-perf-result is private, so raw.githubusercontent.com links to it 404 for +# anyone (GitHub's PR-comment image rendering fetches anonymously - it +# doesn't carry the viewer's session). gh-pages is already public though (it +# already serves the regression history dashboard), so push just the plot +# images - not the raw logs - there too, under their own folder, so the PR +# comment can link to a URL that actually renders. +GHPAGES_DIR=$kit_root/result_repo_ghpages +checkout_result_repo_at $GHPAGES_DIR gh-pages + +IMAGE_DIR=$GHPAGES_DIR/pr-compare/openjdk/pr-$pr_number/$plan_name/$run_id +mkdir -p $IMAGE_DIR +if ls $log_dir/$run_id/plots/*.png >/dev/null 2>&1; then + cp $log_dir/$run_id/plots/*.png $IMAGE_DIR + commit_result_repo_at $GHPAGES_DIR 'OpenJDK compare PR #'$pr_number' ('$plan_name'): '$run_id' images' +fi diff --git a/scripts/openjdk-compare.sh b/scripts/openjdk-compare.sh deleted file mode 100755 index ea3e16e..0000000 --- a/scripts/openjdk-compare.sh +++ /dev/null @@ -1,90 +0,0 @@ -set -ex - -# include common.sh -. $(dirname "$0")/common.sh - -openjdk_binding_trunk=$(realpath $1) -mmtk_core_trunk=$(realpath $2) -openjdk_binding_branch=$(realpath $3) -mmtk_core_branch=$(realpath $4) -output_file=$(realpath -m $5) - -openjdk_trunk_rev=$(git -C $openjdk_binding_trunk rev-parse HEAD) -mmtk_trunk_rev=$(git -C $mmtk_core_trunk rev-parse HEAD) -openjdk_branch_rev=$(git -C $openjdk_binding_branch rev-parse HEAD) -mmtk_branch_rev=$(git -C $mmtk_core_branch rev-parse HEAD) - -# OpenJDK root -openjdk_trunk=$openjdk_binding_trunk/repos/openjdk -openjdk_branch=$openjdk_binding_branch/repos/openjdk - -# Edit openjdk binding Cargo.toml to use local path for mmtk core - note: this makes this script not repeatable -openjdk_binding_use_local_mmtk $openjdk_binding_trunk -if [ "$openjdk_binding_branch" != "$openjdk_binding_trunk" ]; then - openjdk_binding_use_local_mmtk $openjdk_binding_branch -fi - -# Build -ensure_empty_dir $kit_build -ensure_empty_dir $kit_upload -ensure_empty_dir $log_dir - -# Build probes -build_probes - -# Build for trunk -rm -rf $openjdk_binding_trunk/repos/mmtk-core -ln -sfn $mmtk_core_trunk $openjdk_binding_trunk/repos/mmtk-core -build_openjdk_with_mmtk $openjdk_binding_trunk release jdk-mmtk-trunk - -# Build for branch -rm -rf $openjdk_binding_branch/repos/mmtk-core -ln -sfn $mmtk_core_branch $openjdk_binding_branch/repos/mmtk-core -build_openjdk_with_mmtk $openjdk_binding_branch release jdk-mmtk-branch - -# Run -cd $kit_root - -echo "OpenJDK" >> $output_file -echo "====" >> $output_file - -echo "* binding_trunk: [$openjdk_trunk_rev](https://github.com/mmtk/mmtk-openjdk/commit/$openjdk_trunk_rev)" >> $output_file -echo "* trunk: [$mmtk_trunk_rev](https://github.com/mmtk/mmtk-core/commit/$mmtk_trunk_rev)" >> $output_file -echo "* binding_branch: [$openjdk_branch_rev](https://github.com/mmtk/mmtk-openjdk/commit/$openjdk_branch_rev)" >> $output_file -echo "* branch: [$mmtk_branch_rev](https://github.com/mmtk/mmtk-core/commit/$mmtk_branch_rev)" >> $output_file - -echo "" >> $output_file - -# Python venv -start_venv python-env -pip3 install -r scripts/requirements.txt - -run_exp() { - plan=$1 - run_config=$2 - heap_modifier=$3 - - run_id=$(run_benchmarks $log_dir $kit_root/configs/$run_config $heap_modifier $compare_invocations) - python $kit_root/scripts/compare_report.py $log_dir/$run_id $plan jdk-mmtk-trunk jdk-mmtk-branch $compare_invocations >> $output_file -} - -# NoGC -run_exp NoGC running-openjdk-nogc-compare.yml 0 - -# SemiSpace -run_exp SemiSpace running-openjdk-semispace-compare.yml 6 - -# GenCopy -run_exp GenCopy running-openjdk-semispace-compare.yml 6 - -# Immix -run_exp Immix running-openjdk-immix-compare.yml 6 - -# GenImmix -run_exp GenImmix running-openjdk-genimmix-compare.yml 6 - -# StickyImmix -run_exp StickyImmix running-openjdk-stickyimmix-compare.yml 6 - -# MarkSweep -run_exp MarkSweep running-openjdk-marksweep-compare.yml 6