// @ts-check import z from "zod" import os from "node:os" import path from "node:path" import fetch from "node-fetch" import { execa } from "execa" import { binary, command, flag, option } from "cmd-ts" import Url from "cmd-ts/dist/cjs/batteries/url.js" import { run } from "cmd-ts" import fs from "node:fs/promises" import { dedent } from "ts-dedent" const HyperfineResult = z.object({ results: z.array( z.object({ command: z.string(), mean: z.number(), stddev: z.number(), median: z.number(), user: z.number(), system: z.number(), min: z.number(), max: z.number(), times: z.array(z.number()), exit_codes: z.array(z.literal(0)), }) ), }) const BenchyResult = z.object({ data: z.object({ embed: z.object({ small: z.string(), big: z.string(), currentValue: z.number(), lastValue: z.number().optional(), diff: z .object({ value: z.number(), arrowImage: z.string(), }) .optional(), }), }), }) const { HttpUrl } = Url const cmd = command({ name: "run-benchmarks", args: { serverUrl: option({ long: "server-url", type: HttpUrl, defaultValue: () => new URL("https://benchy.hagever.com"), defaultValueIsSerializable: true, }), githubToken: option({ long: "github-token", env: "GITHUB_TOKEN", }), shouldStore: flag({ long: "store", }), }, async handler({ serverUrl, githubToken, shouldStore }) { const repoName = "fnm" const repoOwner = "schniz" const hyperfineResult = await runHyperfine() if (!hyperfineResult.success) { console.error( `Can't run benchmarks: wrong data:`, hyperfineResult.error.issues ) process.exitCode = 1 return } const { results } = hyperfineResult.data const url = new URL("/api/metrics", serverUrl) const trackedKeys = ["median", "max", "mean", "min", "stddev"] const metrics = results .flatMap((result) => { return trackedKeys.map((key) => { return { displayName: `${result.command}/${key}`, value: result[key] * 1000, // everything is in seconds units: "ms", } }) }) .concat([ { displayName: `binary size`, value: await getFilesize(), units: "kb", }, ]) .map((metric) => { return { ...metric, key: `${os.platform()}/${os.arch()}/${metric.displayName}`, } }) const embeds$ = metrics.map(async ({ key, value, displayName, units }) => { const response = await fetch(String(url), { method: "PUT", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ coloring: "lower-is-better", repoOwner, repoName, githubToken, key, value, }), }) if (!response.ok) { throw new Error(`Response is not okay: ${response.status}`) } const { data } = BenchyResult.parse(await response.json()) return { displayName, units, ...data.embed, } }) const embeds = await Promise.all(embeds$) const table = (() => { const rows = embeds .map((data) => { return dedent`
${data.displayName}
${round(data.currentValue, 2)}${data.units}
${round(data.lastValue, 2)}${data.units}
`
}0
"
: dedent`
${data.diff.value > 0 ? "+" : ""}${round(
data.diff.value,
2
)}${data.units}
`
}benchmark | current value | last value | diff | trend |
---|