// @ts-check import z from "zod" import os from "node:os" import path from "node:path" import fetch from "node-fetch" import { execa } from "execa" import { binary, command, flag, option } from "cmd-ts" import Url from "cmd-ts/dist/cjs/batteries/url.js" import { run } from "cmd-ts" import fs from "node:fs/promises" import { dedent } from "ts-dedent" const HyperfineResult = z.object({ results: z.array( z.object({ command: z.string(), mean: z.number(), stddev: z.number(), median: z.number(), user: z.number(), system: z.number(), min: z.number(), max: z.number(), times: z.array(z.number()), exit_codes: z.array(z.literal(0)), }) ), }) const BenchyResult = z.object({ data: z.object({ embed: z.object({ small: z.string(), big: z.string(), currentValue: z.number(), lastValue: z.number().optional(), diff: z .object({ value: z.number(), arrowImage: z.string(), }) .optional(), }), }), }) const { HttpUrl } = Url const cmd = command({ name: "run-benchmarks", args: { serverUrl: option({ long: "server-url", type: HttpUrl, defaultValue: () => new URL("https://benchy.hagever.com"), defaultValueIsSerializable: true, }), githubToken: option({ long: "github-token", env: "GITHUB_TOKEN", }), shouldStore: flag({ long: "store", }), }, async handler({ serverUrl, githubToken, shouldStore }) { const repoName = "fnm" const repoOwner = "schniz" const hyperfineResult = await runHyperfine() if (!hyperfineResult.success) { console.error( `Can't run benchmarks: wrong data:`, hyperfineResult.error.issues ) process.exitCode = 1 return } const { results } = hyperfineResult.data const url = new URL("/api/metrics", serverUrl) const trackedKeys = ["median", "max", "mean", "min", "stddev"] const metrics = results .flatMap((result) => { return trackedKeys.map((key) => { return { displayName: `${result.command}/${key}`, value: result[key] * 1000, // everything is in seconds units: "ms", } }) }) .concat([ { displayName: `binary size`, value: await getFilesize(), units: "kb", }, ]) .map((metric) => { return { ...metric, key: `${os.platform()}/${os.arch()}/${metric.displayName}`, } }) const embeds$ = metrics.map(async ({ key, value, displayName, units }) => { const response = await fetch(String(url), { method: "PUT", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ coloring: "lower-is-better", repoOwner, repoName, githubToken, key, value, }), }) if (!response.ok) { throw new Error(`Response is not okay: ${response.status}`) } const { data } = BenchyResult.parse(await response.json()) return { displayName, units, ...data.embed, } }) const embeds = await Promise.all(embeds$) const table = (() => { const rows = embeds .map((data) => { return dedent` ${data.displayName} ${round(data.currentValue, 2)}${data.units} ${ typeof data.lastValue === "undefined" ? "" : `${round(data.lastValue, 2)}${data.units}` } ${ !data.diff ? "0" : dedent` 0 ? "increase" : "decrease" )}> ${data.diff.value > 0 ? "+" : ""}${round( data.diff.value, 2 )}${data.units} ` }

` }) .join("\n") return dedent` ${rows}
benchmark current value last value diff trend
` })() console.log(table) if (shouldStore) { const response = await fetch(String(url), { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ repoOwner, repoName, githubToken, metrics, }), }) if (!response.ok) { throw new Error(`Response is not okay: ${response.status}`) } console.error(await response.json()) } }, }) /** * @param {number} number * @param {number} digits */ function round(number, digits) { const pow = Math.pow(10, digits) return Math.round(number * pow) / pow } /** * Returns the size of the `fnm` binary in kilobytes * * @returns number */ async function getFilesize() { const fnmBinary = await execa("which", ["fnm"]) const stat = await fs.stat(fnmBinary.stdout.trim()) return Math.round(stat.size / 1024) } async function runHyperfine() { const file = path.join(os.tmpdir(), `bench-${Date.now()}.json`) await execa( `hyperfine`, [ "--min-runs=20", `--export-json=${file}`, "--warmup=2", ...[ "--command-name=fnm_basic", new URL("./basic/fnm", import.meta.url).pathname, ], // ...[ // "--command-name=nvm_basic", // new URL("./basic/nvm", import.meta.url).pathname, // ], ], { stdout: process.stderr, stderr: process.stderr, stdin: "ignore", } ) const json = JSON.parse(await fs.readFile(file, "utf8")) const parsed = HyperfineResult.safeParse(json) return parsed } run(binary(cmd), process.argv)