#!/usr/bin/env python3
"""Build two revisions once, then alternate bounded GPUI benchmark runs.

Usage: script/gpui-bench-compare BASELINE CANDIDATE
       script/gpui-bench-compare BASELINE CANDIDATE --filter 'TextPanels/'

The invoking checkout's three benchmark files, randomized tree fixture and bench
macro are used for BOTH revisions, so both revisions must provide the GPUI
benchmark APIs they call. Sources, build artifacts, binaries, raw logs, Criterion
samples and summary.json persist under target/gpui-bench-compare. Nothing
switches the invoking checkout or publishes a branch.

The default core suite keeps the mixed workload (plain and with cached views),
five full-refresh tree shapes, element style, cached element style, pointer
hover, warm installed-font text, text first frame, three scrolling workloads,
random and patterned primitive painting, and two inspector workloads. --full
selects every variant; --slow restores the benchmarks' own Criterion settings
(including inspector's five-second measurement). --repetitions controls
independent runs, not random seeds. Use --seed to compare another fixed tree.

Foreground retired instructions are primary. Linux needs perf access
(CAP_PERFMON or perf_event_paranoid <= 2); macOS needs Apple Silicon. Frames run
the production pipeline through the platform's headless GPU renderer, so they
include scene encoding and GPU submission; GPU execution itself runs on other
threads and isn't counted. Wall-time medians include calibration samples; raw
logs retain frame metrics. Spread is the full range of independent run medians
divided by their median, not a confidence interval. Warnings also flag wide
within-run sample ranges; even slow settings can drift, so inspect raw samples
before judging small changes. Do not run competing benchmarks on the same machine.
Requires Python 3.12+ on Linux or macOS.
"""

import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import signal
import statistics
import subprocess
import sys
import tarfile
import time


TARGETS = ("randomized_element_tree", "primitives", "inspector_render")
FIXTURE_FILES = tuple(
    Path("crates/gpui_platform/benches") / f"{target}.rs" for target in TARGETS
) + (
    Path("crates/gpui_macros/src/bench.rs"),
    Path("crates/gpui/src/randomized_element_tree.rs"),
)
CORE = {
    "randomized_element_tree": (
        r"^((Randomized|Cached)Tree/mixed/tree/mixed/|"
        r"RandomizedTree/full refresh/tree/|"
        r"RandomizedTree/element style/tree/mixed/|"
        r"CachedTree/element style/tree/mixed/|"
        r"RandomizedTree/hover/tree/mixed/|"
        r"Scroll/workload/|"
        r"TextPanels/installed font two updates/panels/32$|"
        r"TextPanels/first populated installed font/panels/32$)"
    ),
    # Overlapping paint cases, including the random ones with large paths and text,
    # primarily stress GPU overdraw, not draw ordering; GPU queue backpressure makes
    # their instruction counts vary by up to ~16% within a run.
    "primitives": (
        r"^Primitives/(scene/[a-z]+-1024(-seed-\d+)?|paint/case/grid-1024)$"
    ),
    "inspector_render": (
        r"^closed_inspector_(grid/case/1000-nodes|panel/case/500-rows)-0-cycles$"
    ),
}
FAST = [
    "--warm-up-time", "0.05",
    "--measurement-time", "0.3",
    "--sample-size", "10",
    "--nresamples", "1000",
]


def run(command, *, cwd, environment, log, timeout):
    started = time.monotonic()
    with log.open("w") as output:
        # A session of its own lets a timeout or interrupt stop the whole process tree:
        # killing only Cargo would leave its compiler and build-script children running
        # after this script exits and releases the comparison lock.
        process = subprocess.Popen(
            command, cwd=cwd, env=environment, stdout=output,
            stderr=subprocess.STDOUT, start_new_session=True,
        )
        try:
            returncode = process.wait(timeout=timeout)
        except BaseException as error:
            os.killpg(process.pid, signal.SIGKILL)
            process.wait()
            if isinstance(error, subprocess.TimeoutExpired):
                raise RuntimeError(f"{error}; see {log}") from error
            raise
    if returncode != 0:
        raise RuntimeError(f"{command[0]} exited with status {returncode}; see {log}")
    return time.monotonic() - started


def git(root, *arguments):
    return subprocess.check_output(["git", "-C", str(root), *arguments], text=True).strip()


def prepare(root, cache, revision, features, timeout, environment):
    commit = git(root, "rev-parse", "--verify", f"{revision}^{{commit}}")
    source = cache / "sources" / commit
    source.mkdir(parents=True, exist_ok=True)
    if not (source / ".archive-complete").exists():
        archive = cache / f"{commit}.tar"
        with archive.open("wb") as output:
            subprocess.run(["git", "-C", str(root), "archive", commit],
                           stdout=output, check=True, timeout=timeout)
        with tarfile.open(archive) as contents:
            contents.extractall(source, filter="data")
        (source / ".archive-complete").touch()
    digest = hashlib.sha256()
    for relative in FIXTURE_FILES:
        content = (root / relative).read_bytes()
        digest.update(str(relative).encode())
        digest.update(content)
        destination = source / relative
        if not destination.exists() or destination.read_bytes() != content:
            destination.write_bytes(content)
    build_key = hashlib.sha256(
        (commit + digest.hexdigest() + features).encode()
    ).hexdigest()[:20]
    artifacts = cache / "binaries" / build_key
    artifacts.mkdir(parents=True, exist_ok=True)
    # Cargo's workspace fingerprints use mtimes, not source contents. Archives
    # from different commits can otherwise reuse the preceding revision's binary.
    environment = dict(environment, CARGO_TARGET_DIR=str(cache / "build" / commit))
    feature_log = artifacts / "features.log"
    run(
        ["cargo", "tree", "--locked", "-p", "gpui_platform", "--features", features,
         "-e", "normal,build,dev,features"],
        cwd=source, environment=environment, log=feature_log, timeout=timeout,
    )
    if 'feature "test-support"' in feature_log.read_text():
        raise RuntimeError(f"benchmark graph enables test-support: {feature_log}")
    command = [
        "cargo", "bench", "--locked", "-p", "gpui_platform", "--features", features,
        "--profile", "release-fast", "--no-run", "--message-format=json",
    ]
    for target in TARGETS:
        command += ["--bench", target]
    elapsed = run(command, cwd=source, environment=environment,
                  log=artifacts / "build.log", timeout=timeout)
    binaries = {}
    for line in (artifacts / "build.log").read_text().splitlines():
        try:
            message = json.loads(line)
        except json.JSONDecodeError:
            continue
        if message.get("reason") != "compiler-artifact" or not message.get("executable"):
            continue
        name = message["target"]["name"]
        if name in TARGETS:
            # Cargo may overwrite these paths on the next revision's build.
            destination = artifacts / name
            shutil.copy2(message["executable"], destination)
            binaries[name] = destination
    if set(binaries) != set(TARGETS):
        raise RuntimeError(f"missing benchmark executable; see {artifacts / 'build.log'}")
    binary_hashes = {}
    for target, binary in binaries.items():
        with binary.open("rb") as executable:
            binary_hashes[target] = hashlib.file_digest(executable, "sha256").hexdigest()
    return {
        "revision": revision, "commit": commit, "build_seconds": elapsed,
        "gpui_tree": git(root, "rev-parse", f"{commit}:crates/gpui"),
        "binary_sha256": binary_hashes,
        "benchmark_sha256": digest.hexdigest(),
        "lockfile_sha256": hashlib.sha256((source / "Cargo.lock").read_bytes()).hexdigest(),
        "source": str(source), "binaries": binaries,
    }


def verify_distinct_artifacts(revisions):
    baseline, candidate = revisions
    if baseline["gpui_tree"] != candidate["gpui_tree"]:
        for target in TARGETS:
            if baseline["binary_sha256"][target] == candidate["binary_sha256"][target]:
                raise RuntimeError(
                    f"{target} binaries are byte-identical despite different crates/gpui trees; "
                    "refusing a potentially contaminated benchmark comparison"
                )


def measure(binary, identifier, directory, environment, timeout, slow):
    directory.mkdir(parents=True, exist_ok=True)
    environment = dict(environment, CRITERION_HOME=str(directory / "criterion"))
    command = [str(binary), "--bench", "^" + re.escape(identifier) + "$", "--noplot"]
    if not slow:
        command += FAST
    elapsed = run(command, cwd=directory, environment=environment,
                  log=directory / "run.log", timeout=timeout)
    samples = list((directory / "criterion").glob("**/new/sample.json"))
    if len(samples) != 1:
        raise RuntimeError(f"expected one measured case, got {len(samples)}: {directory}")
    sample = json.loads(samples[0].read_text())
    counts = [total / count for total, count in zip(sample["times"], sample["iters"])]
    wall = re.search(
        r"wall time per iteration: median ([\d.]+) (\S+) \(min ([\d.]+), max ([\d.]+)",
        (directory / "run.log").read_text(),
    )
    if wall is None:
        raise RuntimeError(f"missing wall-time secondary: {directory / 'run.log'}")
    units = {"ns": 1e-9, "µs": 1e-6, "us": 1e-6, "ms": 1e-3, "s": 1}
    return {
        "instructions": statistics.median(counts),
        "sample_min": min(counts), "sample_max": max(counts),
        "wall_seconds": float(wall[1]) * units[wall[2]],
        "elapsed_seconds": elapsed, "iterations": sum(sample["iters"]),
    }


def spread(values):
    return (max(values) - min(values)) / statistics.median(values) * 100


def main():
    started = time.monotonic()
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("baseline")
    parser.add_argument("candidate")
    parser.add_argument("--filter", help="Criterion regex, applied to all three binaries")
    parser.add_argument("--full", action="store_true")
    parser.add_argument("--slow", action="store_true")
    parser.add_argument("--repetitions", type=int, default=3)
    parser.add_argument("--seed", type=int, default=0)
    # Text benchmarks need the platform's real text system: x11 on Linux, font-kit on macOS.
    parser.add_argument("--features", default={"linux": "bench-support,x11",
                                                "darwin": "bench-support,font-kit"}.get(
                                                    sys.platform, "bench-support"))
    parser.add_argument("--timeout", type=int, default=300,
                        help="hard timeout for EACH build/run, at most 300 seconds")
    parser.add_argument("--cache", type=Path)
    args = parser.parse_args()
    if not 1 <= args.timeout <= 300 or args.repetitions < 1 or args.seed < 0:
        parser.error("timeout must be 1..300, repetitions positive and seed nonnegative")
    if args.filter and args.full:
        parser.error("--filter and --full are mutually exclusive")
    if "bench-support" not in args.features.split(",") or "test-support" in args.features:
        parser.error("features must include bench-support and must not include test-support")
    root = Path(git(Path(__file__).resolve().parent, "rev-parse", "--show-toplevel"))
    cache = (args.cache or root / "target/gpui-bench-compare").resolve()
    cache.mkdir(parents=True, exist_ok=True)
    lock = (cache / "compare.lock").open("w")
    try:
        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except BlockingIOError as error:
        raise RuntimeError(f"another comparison is using {cache}") from error
    output = cache / "runs" / time.strftime("%Y%m%d-%H%M%S")
    output.mkdir(parents=True, exist_ok=False)
    environment = dict(os.environ, BENCH_MEASUREMENT="foreground-instructions",
                       SEED=str(args.seed), ITERATIONS="1")
    environment["PATH"] = str(Path.home() / ".cargo/bin") + os.pathsep + environment["PATH"]
    revisions = []
    for revision in [args.baseline, args.candidate]:
        print(f"Building {revision} (once)...", flush=True)
        revisions.append(prepare(root, cache, revision, args.features, args.timeout, environment))
    verify_distinct_artifacts(revisions)
    if revisions[0]["lockfile_sha256"] != revisions[1]["lockfile_sha256"]:
        print("NOTE: revisions have different Cargo.lock files; dependency changes may affect results.")
    cases = {}
    for target in TARGETS:
        listings = []
        pattern = args.filter if args.filter else "" if args.full else CORE[target]
        for index, revision in enumerate(revisions):
            log = output / f"list-{index}-{target}.log"
            run([str(revision["binaries"][target]), "--list", pattern],
                cwd=output, environment=environment, log=log, timeout=args.timeout)
            listings.append([line.removesuffix(": benchmark") for line in log.read_text().splitlines()
                             if line.endswith(": benchmark")])
        if listings[0] != listings[1]:
            raise RuntimeError(f"benchmark IDs differ for {target}")
        cases[target] = listings[0]
    if not any(cases.values()):
        raise RuntimeError("filter selected no benchmarks")
    # Verify all selected paths once; exercise Criterion's quick path before sampling.
    for index, revision in enumerate(revisions):
        for target, identifiers in cases.items():
            if not identifiers:
                continue
            binary = revision["binaries"][target]
            pattern = "^(" + "|".join(re.escape(value) for value in identifiers) + ")$"
            for mode, selection in [("--test", pattern),
                                    ("--quick", "^" + re.escape(identifiers[0]) + "$")]:
                run([str(binary), "--bench", selection, mode, "--noplot",
                     "--measurement-time", "0.3"], cwd=output, environment=environment,
                    log=output / f"smoke-{index}-{target}{mode}.log", timeout=args.timeout)
    results = {}
    print("Measuring sequentially; instructions are foreground retired instructions per frame/operation.",
          flush=True)
    print(f"{'benchmark':76} {'base M':>9} {'new M':>9} {'change':>9} "
          f"{'spread B/N':>14} {'wall ms B/N':>17} {'run s B/N':>13}", flush=True)
    for target, identifiers in cases.items():
        for identifier in identifiers:
            pair = [[], []]
            case_number = len(results)
            for repetition in range(args.repetitions):
                for index in ([0, 1] if repetition % 2 == 0 else [1, 0]):
                    revision = revisions[index]
                    directory = output / f"{case_number:03}-{repetition}-{index}"
                    pair[index].append(measure(revision["binaries"][target], identifier, directory,
                                               environment, args.timeout, args.slow))
            results[identifier] = pair
            instruction_runs = [[value["instructions"] for value in side] for side in pair]
            baseline, candidate = map(statistics.median, instruction_runs)
            walls = [statistics.median(value["wall_seconds"] for value in side) * 1000 for side in pair]
            durations = [statistics.median(value["elapsed_seconds"] for value in side) for side in pair]
            noise = (f"{spread(instruction_runs[0]):5.2f}/{spread(instruction_runs[1]):5.2f}%"
                     if args.repetitions > 1 else "           n/a")
            print(f"{identifier:76} {baseline / 1e6:9.3f} {candidate / 1e6:9.3f} "
                  f"{(candidate / baseline - 1) * 100:+8.2f}% "
                  f"{noise} "
                  f"{walls[0]:7.3f}/{walls[1]:7.3f} "
                  f"{durations[0]:5.2f}/{durations[1]:5.2f}", flush=True)
            if args.repetitions > 1 and any(spread(values) > 1 for values in instruction_runs):
                print("  NOTE: >1% instruction spread; confirm this case with --slow before judging a change.",
                      flush=True)
            sample_ranges = [
                max((value["sample_max"] - value["sample_min"]) / value["instructions"] * 100
                    for value in side)
                for side in pair
            ]
            if any(value > 1 for value in sample_ranges):
                print(f"  NOTE: within-run instruction sample range B/N "
                      f"{sample_ranges[0]:.2f}/{sample_ranges[1]:.2f}%; "
                      "inspect raw samples for drift/outliers before judging a change.", flush=True)
            summary = {
                "configuration": vars(args) | {"cache": str(cache)},
                "revisions": [{key: value for key, value in revision.items() if key != "binaries"}
                              for revision in revisions],
                "results": results,
                "cases": cases,
                "environment": {name: value for name, value in environment.items()
                                if name in ("BENCH_MEASUREMENT", "SEED", "ITERATIONS",
                                            "RUSTFLAGS")},
            }
            (output / "summary.json").write_text(json.dumps(summary, indent=2, default=str) + "\n")
    for index, revision in enumerate(revisions):
        seconds = sum(value["elapsed_seconds"] for pair in results.values() for value in pair[index])
        print(f"{revision['commit'][:12]}: build {revision['build_seconds']:.2f}s, "
              f"measured processes {seconds:.2f}s ({args.repetitions} runs/case)")
        for target, identifiers in cases.items():
            seconds = sum(value["elapsed_seconds"] for identifier in identifiers
                          for value in results[identifier][index])
            print(f"  {target}: {seconds:.2f}s")
    print(f"Total including builds, discovery, smoke and quick checks: {time.monotonic() - started:.2f}s")
    print(f"Raw samples, frame reports and summary: {output}")


if __name__ == "__main__":
    try:
        main()
    except (RuntimeError, subprocess.CalledProcessError) as error:
        sys.exit(str(error))
