Browse Source
* wip * install hyperfine * switch sides * add permissions block * permissions * use ternary * wip wip wip * wip * other way? * force store * simplify * add more perms * add file size and units * change columns * add header * add hash * reverse the conditional to store * show last value for 0 * lastValue is now being sent * 0 * the return of all the build * Add min runsremotes/origin/alias-latest
Gal Schlezinger
2 years ago
committed by
GitHub
9 changed files with 458 additions and 6 deletions
@ -0,0 +1,6 @@
@@ -0,0 +1,6 @@
|
||||
#!/bin/bash |
||||
|
||||
eval "$(fnm env --multi)" |
||||
fnm install v10.11.0 |
||||
fnm use v10.11.0 |
||||
node -v |
||||
@ -0,0 +1,6 @@
@@ -0,0 +1,6 @@
|
||||
#!/bin/bash |
||||
|
||||
eval "$(~/.fnm-latest/fnm env --multi)" |
||||
~/.fnm-latest/fnm install v10.11.0 |
||||
~/.fnm-latest/fnm use v10.11.0 |
||||
node -v |
||||
@ -0,0 +1,6 @@
@@ -0,0 +1,6 @@
|
||||
#!/bin/bash |
||||
|
||||
eval "$(~/.fnm/fnm env --multi)" |
||||
~/.fnm/fnm install v10.11.0 |
||||
~/.fnm/fnm use v10.11.0 |
||||
node -v |
||||
@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash |
||||
|
||||
export NVM_DIR="$HOME/.nvm" |
||||
\. "$NVM_DIR/nvm.sh" # This loads nvm |
||||
|
||||
nvm install v10.11.0 |
||||
nvm use v10.11.0 |
||||
node -v |
||||
@ -0,0 +1,36 @@
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash |
||||
|
||||
set -e |
||||
|
||||
export FNM_DIR |
||||
|
||||
BASE_DIR="$(dirname "$(realpath "$0")")" |
||||
cd "$BASE_DIR" || exit 1 |
||||
|
||||
FNM_DIR="$(mktemp -d)" |
||||
export PATH="$BASE_DIR/../target/release:$PATH" |
||||
|
||||
mkdir results 2>/dev/null || : |
||||
|
||||
if [ ! -f "$BASE_DIR/../target/release/fnm" ]; then |
||||
echo "Can't access the release version of fnm.rs" |
||||
exit 1 |
||||
fi |
||||
|
||||
if ! command -v hyperfine >/dev/null 2>&1; then |
||||
echo "Can't access Hyperfine. Are you sure it is installed?" |
||||
echo " if not, visit https://github.com/sharkdp/hyperfine" |
||||
exit 1 |
||||
fi |
||||
|
||||
# Running it with warmup means we're going to have the versions |
||||
# pre-installed. I think it is good because you open your shell more times |
||||
# than you install Node versions. |
||||
hyperfine \ |
||||
--warmup=2 \ |
||||
--min-runs=40 \ |
||||
--time-unit=millisecond \ |
||||
--export-json="./results/basic.json" \ |
||||
--export-markdown="./results/basic.md" \ |
||||
"basic/nvm" \ |
||||
"basic/fnm" |
||||
@ -0,0 +1,272 @@
@@ -0,0 +1,272 @@
|
||||
// @ts-check
|
||||
|
||||
import z from "zod" |
||||
import os from "node:os" |
||||
import path from "node:path" |
||||
import fetch from "node-fetch" |
||||
import { execa } from "execa" |
||||
import { binary, command, flag, option } from "cmd-ts" |
||||
import Url from "cmd-ts/dist/cjs/batteries/url.js" |
||||
import { run } from "cmd-ts" |
||||
import fs from "node:fs/promises" |
||||
import { dedent } from "ts-dedent" |
||||
|
||||
const HyperfineResult = z.object({ |
||||
results: z.array( |
||||
z.object({ |
||||
command: z.string(), |
||||
mean: z.number(), |
||||
stddev: z.number(), |
||||
median: z.number(), |
||||
user: z.number(), |
||||
system: z.number(), |
||||
min: z.number(), |
||||
max: z.number(), |
||||
times: z.array(z.number()), |
||||
exit_codes: z.array(z.literal(0)), |
||||
}) |
||||
), |
||||
}) |
||||
|
||||
const BenchyResult = z.object({ |
||||
data: z.object({ |
||||
embed: z.object({ |
||||
small: z.string(), |
||||
big: z.string(), |
||||
|
||||
currentValue: z.number(), |
||||
lastValue: z.number().optional(), |
||||
diff: z |
||||
.object({ |
||||
value: z.number(), |
||||
arrowImage: z.string(), |
||||
}) |
||||
.optional(), |
||||
}), |
||||
}), |
||||
}) |
||||
|
||||
const { HttpUrl } = Url |
||||
|
||||
const cmd = command({ |
||||
name: "run-benchmarks", |
||||
args: { |
||||
serverUrl: option({ |
||||
long: "server-url", |
||||
type: HttpUrl, |
||||
defaultValue: () => new URL("https://benchy.hagever.com"), |
||||
defaultValueIsSerializable: true, |
||||
}), |
||||
githubToken: option({ |
||||
long: "github-token", |
||||
env: "GITHUB_TOKEN", |
||||
}), |
||||
shouldStore: flag({ |
||||
long: "store", |
||||
}), |
||||
}, |
||||
async handler({ serverUrl, githubToken, shouldStore }) { |
||||
const repoName = "fnm" |
||||
const repoOwner = "schniz" |
||||
|
||||
const hyperfineResult = await runHyperfine() |
||||
|
||||
if (!hyperfineResult.success) { |
||||
console.error( |
||||
`Can't run benchmarks: wrong data:`, |
||||
hyperfineResult.error.issues |
||||
) |
||||
process.exitCode = 1 |
||||
return |
||||
} |
||||
|
||||
const { results } = hyperfineResult.data |
||||
|
||||
const url = new URL("/api/metrics", serverUrl) |
||||
const trackedKeys = ["median", "max", "mean", "min", "stddev"] |
||||
|
||||
const metrics = results |
||||
.flatMap((result) => { |
||||
return trackedKeys.map((key) => { |
||||
return { |
||||
displayName: `${result.command}/${key}`, |
||||
value: result[key] * 1000, // everything is in seconds
|
||||
units: "ms", |
||||
} |
||||
}) |
||||
}) |
||||
.concat([ |
||||
{ |
||||
displayName: `binary size`, |
||||
value: await getFilesize(), |
||||
units: "kb", |
||||
}, |
||||
]) |
||||
.map((metric) => { |
||||
return { |
||||
...metric, |
||||
key: `${os.platform()}/${os.arch()}/${metric.displayName}`, |
||||
} |
||||
}) |
||||
|
||||
const embeds$ = metrics.map(async ({ key, value, displayName, units }) => { |
||||
const response = await fetch(String(url), { |
||||
method: "PUT", |
||||
headers: { |
||||
"Content-Type": "application/json", |
||||
}, |
||||
body: JSON.stringify({ |
||||
coloring: "lower-is-better", |
||||
repoOwner, |
||||
repoName, |
||||
githubToken, |
||||
key, |
||||
value, |
||||
}), |
||||
}) |
||||
|
||||
if (!response.ok) { |
||||
throw new Error(`Response is not okay: ${response.status}`) |
||||
} |
||||
|
||||
const { data } = BenchyResult.parse(await response.json()) |
||||
return { |
||||
displayName, |
||||
units, |
||||
...data.embed, |
||||
} |
||||
}) |
||||
|
||||
const embeds = await Promise.all(embeds$) |
||||
|
||||
const table = (() => { |
||||
const rows = embeds |
||||
.map((data) => { |
||||
return dedent` |
||||
<tr> |
||||
<td><code>${data.displayName}</code></td> |
||||
<td><code>${round(data.currentValue, 2)}${data.units}</code></td> |
||||
<td>${ |
||||
typeof data.lastValue === "undefined" |
||||
? "" |
||||
: `<code>${round(data.lastValue, 2)}${data.units}</code>` |
||||
}</td> |
||||
<td>${ |
||||
!data.diff |
||||
? "<code>0</code>" |
||||
: dedent` |
||||
<picture title=${JSON.stringify( |
||||
data.diff.value > 0 ? "increase" : "decrease" |
||||
)}> |
||||
<img width="16" valign="middle" src="${ |
||||
data.diff.arrowImage |
||||
}"> |
||||
</picture> |
||||
<code>${data.diff.value > 0 ? "+" : ""}${round( |
||||
data.diff.value, |
||||
2 |
||||
)}${data.units}</code> |
||||
` |
||||
}</td> |
||||
<td> |
||||
<details><summary><img valign="middle" src="${ |
||||
data.small |
||||
}" /></summary><br/><img src="${data.big}" /></details> |
||||
</td> |
||||
</tr> |
||||
` |
||||
}) |
||||
.join("\n") |
||||
return dedent` |
||||
<table> |
||||
<thead> |
||||
<tr> |
||||
<th align="left">benchmark</th> |
||||
<th>current value</th> |
||||
<th>last value</th> |
||||
<th>diff</th> |
||||
<th>trend</th> |
||||
</tr> |
||||
</thead> |
||||
<tbody> |
||||
${rows} |
||||
</tbody> |
||||
</table> |
||||
` |
||||
})() |
||||
|
||||
console.log(table) |
||||
|
||||
if (shouldStore) { |
||||
const response = await fetch(String(url), { |
||||
method: "POST", |
||||
headers: { |
||||
"Content-Type": "application/json", |
||||
}, |
||||
body: JSON.stringify({ |
||||
repoOwner, |
||||
repoName, |
||||
githubToken, |
||||
metrics, |
||||
}), |
||||
}) |
||||
|
||||
if (!response.ok) { |
||||
throw new Error(`Response is not okay: ${response.status}`) |
||||
} |
||||
|
||||
console.error(await response.json()) |
||||
} |
||||
}, |
||||
}) |
||||
|
||||
/** |
||||
* @param {number} number |
||||
* @param {number} digits |
||||
*/ |
||||
function round(number, digits) { |
||||
const pow = Math.pow(10, digits) |
||||
return Math.round(number * pow) / pow |
||||
} |
||||
|
||||
/** |
||||
* Returns the size of the `fnm` binary in kilobytes |
||||
* |
||||
* @returns number |
||||
*/ |
||||
async function getFilesize() { |
||||
const fnmBinary = await execa("which", ["fnm"]) |
||||
const stat = await fs.stat(fnmBinary.stdout.trim()) |
||||
return Math.round(stat.size / 1024) |
||||
} |
||||
|
||||
async function runHyperfine() { |
||||
const file = path.join(os.tmpdir(), `bench-${Date.now()}.json`) |
||||
await execa( |
||||
`hyperfine`, |
||||
[ |
||||
"--min-runs=20", |
||||
`--export-json=${file}`, |
||||
"--warmup=2", |
||||
...[ |
||||
"--command-name=fnm_basic", |
||||
new URL("./basic/fnm", import.meta.url).pathname, |
||||
], |
||||
// ...[
|
||||
// "--command-name=nvm_basic",
|
||||
// new URL("./basic/nvm", import.meta.url).pathname,
|
||||
// ],
|
||||
], |
||||
{ |
||||
stdout: process.stderr, |
||||
stderr: process.stderr, |
||||
stdin: "ignore", |
||||
} |
||||
) |
||||
|
||||
const json = JSON.parse(await fs.readFile(file, "utf8")) |
||||
const parsed = HyperfineResult.safeParse(json) |
||||
return parsed |
||||
} |
||||
|
||||
run(binary(cmd), process.argv) |
||||
Loading…
Reference in new issue