Commit 1c53d79d authored by Gaetano Scandariato's avatar Gaetano Scandariato
Browse files

chore: rerun benchmarks with the fused loop and circular orbits, script the throughput table 

Runs of 2026-10-02 (3 slice steps, before the default became 4): every nested-sampling run on both
machines, the calibration with 24 seeds per loop, throughput and scaling rows (v2) by script.

Co-Authored-By: default avatarClaude Opus 5.5 <noreply@anthropic.com>
parent 902f9a57

paper_benchmarks/gpu_precision/results/summary.json

+70 −77
Changes for paper_benchmarks/gpu_precision/results/summary.json: 70 added lines, 77 removed lines.
Original line number Diff line number Diff line
@@ -6,170 +6,163 @@ 
    "software": "JAX 0.11.0, NumPyro 0.22.0, BlackJAX 1.6.2, jaxoplanet 0.1.0"

  },

  "k2140_likelihood_throughput": {

  "script": "scripts/prof.py (the vectorised potential energy of the K2-140 default configuration), one process per row",

    "script": "scripts/throughput.py (runs scripts/prof.py once per row) and scripts/merge_throughput.py",

    "model": "K2-140 joint model: 2232 K2 points x 7 supersamples + 98 LCOGT + 31 RVs, 20 free parameters",

    "note": "time per call of the vectorised potential energy; evaluations/s = batch / time",

    "rows": [

      {

    "device": "gpu",

        "device": "cpu",

        "precision": "float32",

        "batch": 1,

    "ms": 0.13,

    "evals_per_s": 7692

        "ms": 0.78,

        "evals_per_s": 1282

      },

      {

    "device": "gpu",

        "device": "cpu",

        "precision": "float32",

        "batch": 58,

    "ms": 0.45,

    "evals_per_s": 128889

        "ms": 6.16,

        "evals_per_s": 9416

      },

      {

    "device": "gpu",

        "device": "cpu",

        "precision": "float32",

        "batch": 512,

    "ms": 3.03,

    "evals_per_s": 168977

        "ms": 73.84,

        "evals_per_s": 6934

      },

      {

    "device": "gpu",

        "device": "cpu",

        "precision": "mixed",

        "batch": 1,

    "ms": 0.17,

    "evals_per_s": 5882

        "ms": 0.76,

        "evals_per_s": 1316

      },

      {

    "device": "gpu",

        "device": "cpu",

        "precision": "mixed",

        "batch": 58,

    "ms": 0.53,

    "evals_per_s": 109434

        "ms": 6.2,

        "evals_per_s": 9355

      },

      {

    "device": "gpu",

        "device": "cpu",

        "precision": "mixed",

        "batch": 512,

    "ms": 3.5,

    "evals_per_s": 146286

        "ms": 51.01,

        "evals_per_s": 10037

      },

      {

    "device": "cpu",

        "device": "gpu",

        "precision": "float32",

        "batch": 1,

    "ms": 0.99,

    "evals_per_s": 1010,

    "cpu_devices": 1

        "ms": 0.11,

        "evals_per_s": 9091

      },

      {

    "device": "cpu",

        "device": "gpu",

        "precision": "float32",

        "batch": 58,

    "ms": 8.99,

    "evals_per_s": 6452,

    "cpu_devices": 58

        "ms": 0.39,

        "evals_per_s": 148718

      },

      {

    "device": "cpu",

        "device": "gpu",

        "precision": "float32",

        "batch": 512,

    "ms": 77.67,

    "evals_per_s": 6592,

    "cpu_devices": 64

        "ms": 2.61,

        "evals_per_s": 196169

      },

      {

    "device": "cpu",

        "device": "gpu",

        "precision": "mixed",

        "batch": 1,

    "ms": 1.12,

    "evals_per_s": 893,

    "cpu_devices": 1

        "ms": 0.14,

        "evals_per_s": 7143

      },

      {

    "device": "cpu",

        "device": "gpu",

        "precision": "mixed",

        "batch": 58,

    "ms": 9.65,

    "evals_per_s": 6010,

    "cpu_devices": 58

        "ms": 0.52,

        "evals_per_s": 111538

      },

      {

    "device": "cpu",

        "device": "gpu",

        "precision": "mixed",

        "batch": 512,

    "ms": 78.57,

    "evals_per_s": 6516,

    "cpu_devices": 64

        "ms": 2.97,

        "evals_per_s": 172391

      },

      {

        "device": "gpu",

    "gpu": "H100 NVL",

        "precision": "float32",

        "batch": 1,

    "ms": 0.11,

    "evals_per_s": 9091

        "ms": 0.12,

        "evals_per_s": 8333,

        "gpu": "H100 NVL"

      },

      {

        "device": "gpu",

    "gpu": "H100 NVL",

        "precision": "float32",

        "batch": 58,

    "ms": 0.2,

    "evals_per_s": 290000

        "ms": 0.17,

        "evals_per_s": 341176,

        "gpu": "H100 NVL"

      },

      {

        "device": "gpu",

    "gpu": "H100 NVL",

        "precision": "float32",

        "batch": 512,

    "ms": 0.81,

    "evals_per_s": 632099

        "ms": 0.65,

        "evals_per_s": 787692,

        "gpu": "H100 NVL"

      },

      {

        "device": "gpu",

    "gpu": "H100 NVL",

        "precision": "mixed",

        "batch": 1,

    "ms": 0.14,

    "evals_per_s": 7143

        "ms": 0.15,

        "evals_per_s": 6667,

        "gpu": "H100 NVL"

      },

      {

        "device": "gpu",

    "gpu": "H100 NVL",

        "precision": "mixed",

        "batch": 58,

    "ms": 0.22,

    "evals_per_s": 263636

        "ms": 0.2,

        "evals_per_s": 290000,

        "gpu": "H100 NVL"

      },

      {

        "device": "gpu",

    "gpu": "H100 NVL",

        "precision": "mixed",

        "batch": 512,

    "ms": 0.82,

    "evals_per_s": 624390

        "ms": 0.7,

        "evals_per_s": 731429,

        "gpu": "H100 NVL"

      }

    ],

    "juliet_reference": "juliet/dynesty with batman, each at its best count of logical CPUs: laptop 20 threads, 1,064,748 likelihood calls in 118.9 s (9.0e3 per second); server 60 threads, 1,079,640 in 63.5 s (1.7e4 per second) (cpu_scaling/juliet, cpu_scaling/juliet_ganymede)",

    "measured": {

   "date": "2026-09-29",

   "pyaetna_commit": "b3da492 (plus uncommitted paper-benchmark scripts only)",

   "cpu_threads": 16,

   "cpu_note": "taskset -c 0-15",

   "gpu": "NVIDIA RTX 5070 Ti Laptop",

   "precision_note": "'mixed' = use_float64 true: float64 ephemerides, float32 physical models (the float32 firewall)"

      "laptop": {

        "gpu": "NVIDIA GeForce RTX 5070 Ti Laptop GPU",

        "cpu_taskset": "0-15",

        "provenance": {

          "pyaetna_commit": "ff499a6",

          "src_diff_sha256": "4aaf4307052067fb",

          "date": "2026-10-02 03:32"

        }

      },

  "superseded": "Rows measured before the float32 firewall (2026-09-25, float64 promoted inside the physical models) were removed; the development history keeps them.",

  "measured_cpu": {

   "date": "2026-09-30",

      "server": {

        "gpu": "NVIDIA H100 NVL",

        "provenance": {

          "pyaetna_commit": "9cd57eb",

   "cpu_threads": 16,

   "note": "CPU rows re-measured on 2026-09-30 11:12 with the laptop in the performance power profile on mains (XLA_FLAGS=--xla_force_host_platform_device_count=64, taskset -c 0-15); the rows of 2026-09-29 agreed within 10 %. Raw outputs: PyAETNA_bench_work/runs/prof_cpu_*.out."

          "src_diff_sha256": "c167ec055bea4347",

          "date": "2026-10-02 03:16"

        }

      },

  "measured_h100": {

   "date": "2026-09-30",

   "pyaetna_commit": "9cd57eb",

   "machine": "ExoREC-Ganymede, NVIDIA H100 NVL 94 GB",

   "note": "rows with gpu = 'H100 NVL'; the other GPU rows are the laptop RTX 5070 Ti (2026-09-29)."

      "precision_note": "'mixed' = use_float64 true: float64 ephemerides, float32 physical models (the float32 firewall)"

    }

  },

  "float32_firewall": {

paper_benchmarks/gpu_precision/results/throughput_laptop.json

0 → 100644
+96 −0
Changes for paper_benchmarks/gpu_precision/results/throughput_laptop.json: 96 added lines, 0 removed lines.
Original line number Diff line number Diff line 
{

 "machine": "laptop",

 "gpu": "NVIDIA GeForce RTX 5070 Ti Laptop GPU",

 "cpu_taskset": "0-15",

 "provenance": {

  "pyaetna_commit": "ff499a6",

  "src_diff_sha256": "4aaf4307052067fb",

  "date": "2026-10-02 03:32"

 },

 "rows": [

  {

   "device": "cpu",

   "precision": "float32",

   "batch": 1,

   "ms": 0.78,

   "evals_per_s": 1282

  },

  {

   "device": "cpu",

   "precision": "float32",

   "batch": 58,

   "ms": 6.16,

   "evals_per_s": 9416

  },

  {

   "device": "cpu",

   "precision": "float32",

   "batch": 512,

   "ms": 73.84,

   "evals_per_s": 6934

  },

  {

   "device": "cpu",

   "precision": "mixed",

   "batch": 1,

   "ms": 0.76,

   "evals_per_s": 1316

  },

  {

   "device": "cpu",

   "precision": "mixed",

   "batch": 58,

   "ms": 6.2,

   "evals_per_s": 9355

  },

  {

   "device": "cpu",

   "precision": "mixed",

   "batch": 512,

   "ms": 51.01,

   "evals_per_s": 10037

  },

  {

   "device": "gpu",

   "precision": "float32",

   "batch": 1,

   "ms": 0.11,

   "evals_per_s": 9091

  },

  {

   "device": "gpu",

   "precision": "float32",

   "batch": 58,

   "ms": 0.39,

   "evals_per_s": 148718

  },

  {

   "device": "gpu",

   "precision": "float32",

   "batch": 512,

   "ms": 2.61,

   "evals_per_s": 196169

  },

  {

   "device": "gpu",

   "precision": "mixed",

   "batch": 1,

   "ms": 0.14,

   "evals_per_s": 7143

  },

  {

   "device": "gpu",

   "precision": "mixed",

   "batch": 58,

   "ms": 0.52,

   "evals_per_s": 111538

  },

  {

   "device": "gpu",

   "precision": "mixed",

   "batch": 512,

   "ms": 2.97,

   "evals_per_s": 172391

  }

 ]

}
 
 No newline at end of file

paper_benchmarks/gpu_precision/results/throughput_server.json

0 → 100644
+54 −0
Changes for paper_benchmarks/gpu_precision/results/throughput_server.json: 54 added lines, 0 removed lines.
Original line number Diff line number Diff line 
{

 "machine": "server",

 "gpu": "NVIDIA H100 NVL",

 "cpu_taskset": "0-15",

 "provenance": {

  "pyaetna_commit": "9cd57eb",

  "src_diff_sha256": "c167ec055bea4347",

  "date": "2026-10-02 03:16"

 },

 "rows": [

  {

   "device": "gpu",

   "precision": "float32",

   "batch": 1,

   "ms": 0.12,

   "evals_per_s": 8333

  },

  {

   "device": "gpu",

   "precision": "float32",

   "batch": 58,

   "ms": 0.17,

   "evals_per_s": 341176

  },

  {

   "device": "gpu",

   "precision": "float32",

   "batch": 512,

   "ms": 0.65,

   "evals_per_s": 787692

  },

  {

   "device": "gpu",

   "precision": "mixed",

   "batch": 1,

   "ms": 0.15,

   "evals_per_s": 6667

  },

  {

   "device": "gpu",

   "precision": "mixed",

   "batch": 58,

   "ms": 0.2,

   "evals_per_s": 290000

  },

  {

   "device": "gpu",

   "precision": "mixed",

   "batch": 512,

   "ms": 0.7,

   "evals_per_s": 731429

  }

 ]

}
 
 No newline at end of file

paper_benchmarks/gpu_precision/scripts/merge_throughput.py

0 → 100644
+34 −0
Changes for paper_benchmarks/gpu_precision/scripts/merge_throughput.py: 34 added lines, 0 removed lines.
Original line number Diff line number Diff line 
"""Put the rows of results/throughput_laptop.json and throughput_server.json into results/summary.json.



Replaces the hand-transcribed rows of ``k2140_likelihood_throughput``: laptop CPU and GPU rows from the laptop file,

H100 rows (``gpu = 'H100 NVL'``, as make_tables.py expects) from the server file.

"""



from __future__ import annotations



import json

from pathlib import Path



R = Path(__file__).resolve().parent.parent / "results"

summary = json.loads((R / "summary.json").read_text())

laptop = json.loads((R / "throughput_laptop.json").read_text())

server = json.loads((R / "throughput_server.json").read_text())

rows = [r for r in laptop["rows"] if not r.get("failed")]

for r in server["rows"]:

    if r.get("failed") or r["device"] != "gpu":

        continue

    rows.append(dict(r, gpu="H100 NVL"))

t = summary["k2140_likelihood_throughput"]

t["rows"] = rows

t["script"] = (

    "scripts/throughput.py (runs scripts/prof.py once per row) and scripts/merge_throughput.py"

)

t["measured"] = {

    "laptop": {k: laptop[k] for k in ("gpu", "cpu_taskset", "provenance")},

    "server": {k: server[k] for k in ("gpu", "provenance")},

    "precision_note": "'mixed' = use_float64 true: float64 ephemerides, float32 physical models (the float32 firewall)",

}

for k in ("measured_cpu", "measured_h100", "superseded"):

    t.pop(k, None)

(R / "summary.json").write_text(json.dumps(summary, indent=2))

print(f"{len(rows)} rows")

paper_benchmarks/gpu_precision/scripts/throughput.py

0 → 100644
+107 −0
Changes for paper_benchmarks/gpu_precision/scripts/throughput.py: 107 added lines, 0 removed lines.
Original line number Diff line number Diff line 
"""K2-140 likelihood throughput (Table `throughput`), measured and written by script, one process per row.



Runs ``prof.py`` (the vectorised potential energy of the K2-140 default configuration) in a scratch folder laid out

like ``k2140_juliet/runs/`` for every device, precision and batch of the table, parses its time per batched call and

writes ``results/throughput_<machine>.json`` with the provenance of the code. ``merge_throughput.py`` puts the rows

of both machines into ``results/summary.json``.



usage: python throughput.py <machine label> <devices: cpu,gpu> [cpu list for taskset, default 0-15]

"""



from __future__ import annotations



import json

import os

import re

import shutil

import subprocess

import sys

import tempfile

import time

from pathlib import Path



HERE = Path(__file__).resolve().parent

B = HERE.parent.parent  # paper_benchmarks

machine, devices = sys.argv[1], sys.argv[2].split(",")

cpus = sys.argv[3] if len(sys.argv) > 3 else "0-15"

BATCHES = (1, 58, 512)





def provenance() -> dict:

    git = ["git", "-C", str(B.parent)]

    head = subprocess.run(

        git + ["rev-parse", "--short", "HEAD"], capture_output=True, text=True

    ).stdout.strip()

    diff = subprocess.run(

        git + ["diff", "HEAD", "--", "src"], capture_output=True, text=True

    ).stdout

    import hashlib



    return {

        "pyaetna_commit": head or "unknown",

        "src_diff_sha256": (

            hashlib.sha256(diff.encode()).hexdigest()[:16] if diff else None

        ),

        "date": time.strftime("%Y-%m-%d %H:%M"),

    }





rows = []

with tempfile.TemporaryDirectory(dir=os.environ.get("THROUGHPUT_TMP", str(B))) as tmp:

    run = B / "k2140_juliet" / "runs" / "k2c_e8_rtx"

    shutil.copy(run / "k2140.yaml", tmp)

    for f in ("k2140c.dat", "k2140.dat"):

        shutil.copy(B / "k2140_juliet" / "data" / f, tmp)

    for device in devices:

        for precision, mode in (("float32", "32"), ("mixed", "64")):

            for batch in BATCHES:

                env = dict(

                    os.environ, JAX_PLATFORMS="cpu" if device == "cpu" else "cuda"

                )

                cmd = [sys.executable, str(HERE / "prof.py"), mode, str(batch)]

                if device == "cpu":

                    env["XLA_FLAGS"] = "--xla_force_host_platform_device_count=64"

                    cmd = ["taskset", "-c", cpus] + cmd

                out = subprocess.run(

                    cmd, cwd=tmp, env=env, capture_output=True, text=True

                ).stdout

                m = re.search(r"time per batched eval ([0-9.]+) ms", out)

                if not m:

                    rows.append(

                        {

                            "device": device,

                            "precision": precision,

                            "batch": batch,

                            "failed": True,

                        }

                    )

                    continue

                ms = float(m.group(1))

                rows.append(

                    {

                        "device": device,

                        "precision": precision,

                        "batch": batch,

                        "ms": ms,

                        "evals_per_s": round(batch / ms * 1e3),

                    }

                )

                print(json.dumps(rows[-1]), flush=True)



gpu_name = ""

if "gpu" in devices:

    gpu_name = subprocess.run(

        ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"],

        capture_output=True,

        text=True,

    ).stdout.strip()

out = {

    "machine": machine,

    "gpu": gpu_name,

    "cpu_taskset": cpus,

    "provenance": provenance(),

    "rows": rows,

}

(HERE.parent / "results" / f"throughput_{machine}.json").write_text(

    json.dumps(out, indent=1)

)