diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 01dd57a..0ec3c49 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -106,7 +106,7 @@ jobs: run_install: false - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'pnpm' - name: Get pnpm store directory id: pnpm-cache @@ -140,7 +140,7 @@ jobs: run_install: false - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'pnpm' - name: Get pnpm store directory id: pnpm-cache @@ -176,7 +176,7 @@ jobs: # run_install: false # - uses: actions/setup-node@v3 # with: - # node-version: 16.x + # node-version: 18.x # cache: 'pnpm' # - name: Get pnpm store directory # id: pnpm-cache @@ -212,7 +212,7 @@ jobs: run_install: false - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'pnpm' - name: Get pnpm store directory id: pnpm-cache @@ -340,7 +340,7 @@ jobs: run_install: false - uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: 18.x cache: 'pnpm' - name: Get pnpm store directory id: pnpm-cache @@ -357,3 +357,72 @@ jobs: - name: Generate command markdown run: | pnpm run generate-command-docs --check --binary-path=$(which fnm) + + run_e2e_benchmarks: + runs-on: ubuntu-latest + name: bench/linux + needs: [build_static_linux_binary] + permissions: + contents: write + pull-requests: write + steps: + - name: install necessary shells + run: sudo apt-get update && sudo apt-get install -y fish zsh bash hyperfine + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: fnm-linux + path: target/release + - name: mark binary as executable + run: chmod +x target/release/fnm + - name: install fnm as binary + run: | + sudo install target/release/fnm /bin + fnm --version + - uses: pnpm/action-setup@v2.2.4 + with: + run_install: false + - uses: actions/setup-node@v3 + with: + node-version: 18.x + cache: 'pnpm' + - name: Get pnpm store directory + id: pnpm-cache + run: | + echo "::set-output name=pnpm_cache_dir::$(pnpm store path)" + - uses: actions/cache@v3 + name: Setup pnpm cache + with: + path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + - run: pnpm install + - name: Run benchmarks + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SHOULD_STORE: ${{ toJson(!github.event.pull_request) }} + id: benchmark + run: | + delimiter="$(openssl rand -hex 8)" + echo "markdown<<${delimiter}" >> "${GITHUB_OUTPUT}" + node benchmarks/run.mjs --store=$SHOULD_STORE >> "${GITHUB_OUTPUT}" + echo "${delimiter}" >> "${GITHUB_OUTPUT}" + + - name: Create a PR comment + if: ${{ github.event.pull_request }} + uses: thollander/actions-comment-pull-request@v1 + with: + message: | + + ## Linux Benchmarks for ${{ github.event.pull_request.head.sha }} + ${{ steps.benchmark.outputs.markdown }} + comment_includes: '' + + - name: Create a commit comment + if: ${{ !github.event.pull_request }} + uses: peter-evans/commit-comment@v2 + with: + body: | + ## Linux Benchmarks + ${{ steps.benchmark.outputs.markdown }} diff --git a/benchmarks/basic/fnm b/benchmarks/basic/fnm new file mode 100755 index 0000000..3f25cfc --- /dev/null +++ b/benchmarks/basic/fnm @@ -0,0 +1,6 @@ +#!/bin/bash + +eval "$(fnm env --multi)" +fnm install v10.11.0 +fnm use v10.11.0 +node -v diff --git a/benchmarks/basic/fnm_latest_master b/benchmarks/basic/fnm_latest_master new file mode 100755 index 0000000..c3f0f68 --- /dev/null +++ b/benchmarks/basic/fnm_latest_master @@ -0,0 +1,6 @@ +#!/bin/bash + +eval "$(~/.fnm-latest/fnm env --multi)" +~/.fnm-latest/fnm install v10.11.0 +~/.fnm-latest/fnm use v10.11.0 +node -v diff --git a/benchmarks/basic/fnm_reason b/benchmarks/basic/fnm_reason new file mode 100755 index 0000000..7833821 --- /dev/null +++ b/benchmarks/basic/fnm_reason @@ -0,0 +1,6 @@ +#!/bin/bash + +eval "$(~/.fnm/fnm env --multi)" +~/.fnm/fnm install v10.11.0 +~/.fnm/fnm use v10.11.0 +node -v diff --git a/benchmarks/basic/nvm b/benchmarks/basic/nvm new file mode 100755 index 0000000..daf10a6 --- /dev/null +++ b/benchmarks/basic/nvm @@ -0,0 +1,8 @@ +#!/bin/bash + +export NVM_DIR="$HOME/.nvm" +\. "$NVM_DIR/nvm.sh" # This loads nvm + +nvm install v10.11.0 +nvm use v10.11.0 +node -v diff --git a/benchmarks/run b/benchmarks/run new file mode 100755 index 0000000..20314e0 --- /dev/null +++ b/benchmarks/run @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +export FNM_DIR + +BASE_DIR="$(dirname "$(realpath "$0")")" +cd "$BASE_DIR" || exit 1 + +FNM_DIR="$(mktemp -d)" +export PATH="$BASE_DIR/../target/release:$PATH" + +mkdir results 2>/dev/null || : + +if [ ! -f "$BASE_DIR/../target/release/fnm" ]; then + echo "Can't access the release version of fnm.rs" + exit 1 +fi + +if ! command -v hyperfine >/dev/null 2>&1; then + echo "Can't access Hyperfine. Are you sure it is installed?" + echo " if not, visit https://github.com/sharkdp/hyperfine" + exit 1 +fi + +# Running it with warmup means we're going to have the versions +# pre-installed. I think it is good because you open your shell more times +# than you install Node versions. +hyperfine \ + --warmup=2 \ + --min-runs=40 \ + --time-unit=millisecond \ + --export-json="./results/basic.json" \ + --export-markdown="./results/basic.md" \ + "basic/nvm" \ + "basic/fnm" diff --git a/benchmarks/run.mjs b/benchmarks/run.mjs new file mode 100644 index 0000000..f5b29b1 --- /dev/null +++ b/benchmarks/run.mjs @@ -0,0 +1,272 @@ +// @ts-check + +import z from "zod" +import os from "node:os" +import path from "node:path" +import fetch from "node-fetch" +import { execa } from "execa" +import { binary, command, flag, option } from "cmd-ts" +import Url from "cmd-ts/dist/cjs/batteries/url.js" +import { run } from "cmd-ts" +import fs from "node:fs/promises" +import { dedent } from "ts-dedent" + +const HyperfineResult = z.object({ + results: z.array( + z.object({ + command: z.string(), + mean: z.number(), + stddev: z.number(), + median: z.number(), + user: z.number(), + system: z.number(), + min: z.number(), + max: z.number(), + times: z.array(z.number()), + exit_codes: z.array(z.literal(0)), + }) + ), +}) + +const BenchyResult = z.object({ + data: z.object({ + embed: z.object({ + small: z.string(), + big: z.string(), + + currentValue: z.number(), + lastValue: z.number().optional(), + diff: z + .object({ + value: z.number(), + arrowImage: z.string(), + }) + .optional(), + }), + }), +}) + +const { HttpUrl } = Url + +const cmd = command({ + name: "run-benchmarks", + args: { + serverUrl: option({ + long: "server-url", + type: HttpUrl, + defaultValue: () => new URL("https://benchy.hagever.com"), + defaultValueIsSerializable: true, + }), + githubToken: option({ + long: "github-token", + env: "GITHUB_TOKEN", + }), + shouldStore: flag({ + long: "store", + }), + }, + async handler({ serverUrl, githubToken, shouldStore }) { + const repoName = "fnm" + const repoOwner = "schniz" + + const hyperfineResult = await runHyperfine() + + if (!hyperfineResult.success) { + console.error( + `Can't run benchmarks: wrong data:`, + hyperfineResult.error.issues + ) + process.exitCode = 1 + return + } + + const { results } = hyperfineResult.data + + const url = new URL("/api/metrics", serverUrl) + const trackedKeys = ["median", "max", "mean", "min", "stddev"] + + const metrics = results + .flatMap((result) => { + return trackedKeys.map((key) => { + return { + displayName: `${result.command}/${key}`, + value: result[key] * 1000, // everything is in seconds + units: "ms", + } + }) + }) + .concat([ + { + displayName: `binary size`, + value: await getFilesize(), + units: "kb", + }, + ]) + .map((metric) => { + return { + ...metric, + key: `${os.platform()}/${os.arch()}/${metric.displayName}`, + } + }) + + const embeds$ = metrics.map(async ({ key, value, displayName, units }) => { + const response = await fetch(String(url), { + method: "PUT", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + coloring: "lower-is-better", + repoOwner, + repoName, + githubToken, + key, + value, + }), + }) + + if (!response.ok) { + throw new Error(`Response is not okay: ${response.status}`) + } + + const { data } = BenchyResult.parse(await response.json()) + return { + displayName, + units, + ...data.embed, + } + }) + + const embeds = await Promise.all(embeds$) + + const table = (() => { + const rows = embeds + .map((data) => { + return dedent` + + ${data.displayName} + ${round(data.currentValue, 2)}${data.units} + ${ + typeof data.lastValue === "undefined" + ? "" + : `${round(data.lastValue, 2)}${data.units}` + } + ${ + !data.diff + ? "0" + : dedent` + 0 ? "increase" : "decrease" + )}> + + + ${data.diff.value > 0 ? "+" : ""}${round( + data.diff.value, + 2 + )}${data.units} + ` + } + +

+ + + ` + }) + .join("\n") + return dedent` + + + + + + + + + + + + ${rows} + +
benchmarkcurrent valuelast valuedifftrend
+ ` + })() + + console.log(table) + + if (shouldStore) { + const response = await fetch(String(url), { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + repoOwner, + repoName, + githubToken, + metrics, + }), + }) + + if (!response.ok) { + throw new Error(`Response is not okay: ${response.status}`) + } + + console.error(await response.json()) + } + }, +}) + +/** + * @param {number} number + * @param {number} digits + */ +function round(number, digits) { + const pow = Math.pow(10, digits) + return Math.round(number * pow) / pow +} + +/** + * Returns the size of the `fnm` binary in kilobytes + * + * @returns number + */ +async function getFilesize() { + const fnmBinary = await execa("which", ["fnm"]) + const stat = await fs.stat(fnmBinary.stdout.trim()) + return Math.round(stat.size / 1024) +} + +async function runHyperfine() { + const file = path.join(os.tmpdir(), `bench-${Date.now()}.json`) + await execa( + `hyperfine`, + [ + "--min-runs=20", + `--export-json=${file}`, + "--warmup=2", + ...[ + "--command-name=fnm_basic", + new URL("./basic/fnm", import.meta.url).pathname, + ], + // ...[ + // "--command-name=nvm_basic", + // new URL("./basic/nvm", import.meta.url).pathname, + // ], + ], + { + stdout: process.stderr, + stderr: process.stderr, + stdin: "ignore", + } + ) + + const json = JSON.parse(await fs.readFile(file, "utf8")) + const parsed = HyperfineResult.safeParse(json) + return parsed +} + +run(binary(cmd), process.argv) diff --git a/package.json b/package.json index 53611c2..026f575 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "execa": "6.1.0", "jest": "^29.3.1", "lerna-changelog": "2.2.0", + "node-fetch": "^3.3.0", "prettier": "2.8.1", "pv": "1.0.1", "shell-escape": "^0.2.0", @@ -40,7 +41,8 @@ "toml": "3.0.0", "ts-dedent": "^2.2.0", "ts-jest": "^29.0.3", - "typescript": "^4.8.4" + "typescript": "^4.8.4", + "zod": "^3.19.1" }, "prettier": { "semi": false diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dd9780d..8267b5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,7 @@ specifiers: execa: 6.1.0 jest: ^29.3.1 lerna-changelog: 2.2.0 + node-fetch: ^3.3.0 prettier: 2.8.1 pv: 1.0.1 shell-escape: ^0.2.0 @@ -20,6 +21,7 @@ specifiers: ts-dedent: ^2.2.0 ts-jest: ^29.0.3 typescript: ^4.8.4 + zod: ^3.19.1 devDependencies: '@changesets/cli': 2.25.2 @@ -33,6 +35,7 @@ devDependencies: execa: 6.1.0 jest: 29.3.1_@types+node@18.11.9 lerna-changelog: 2.2.0 + node-fetch: 3.3.0 prettier: 2.8.1 pv: 1.0.1 shell-escape: 0.2.0 @@ -41,6 +44,7 @@ devDependencies: ts-dedent: 2.2.0 ts-jest: 29.0.3_4f6uxrzmuwipl5rr3bcogf6k74 typescript: 4.9.3 + zod: 3.19.1 packages: @@ -1842,6 +1846,11 @@ packages: array-find-index: 1.0.2 dev: true + /data-uri-to-buffer/4.0.0: + resolution: {integrity: sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==} + engines: {node: '>= 12'} + dev: true + /dataloader/1.4.0: resolution: {integrity: sha512-68s5jYdlvasItOJnCuI2Q9s4q98g0pCyL3HrcKJu8KNugUl8ahgmZYg38ysLTgQjjXX3H8CJLkAvWrclWfcalw==} dev: true @@ -2173,6 +2182,14 @@ packages: bser: 2.1.1 dev: true + /fetch-blob/3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.2.1 + dev: true + /fill-range/7.0.1: resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} engines: {node: '>=8'} @@ -2211,6 +2228,13 @@ packages: pkg-dir: 4.2.0 dev: true + /formdata-polyfill/4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.2.0 + dev: true + /fs-extra/7.0.1: resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==} engines: {node: '>=6 <7 || >=8'} @@ -3595,6 +3619,11 @@ packages: engines: {node: '>= 0.6'} dev: true + /node-domexception/1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: true + /node-fetch/1.7.3: resolution: {integrity: sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==} dependencies: @@ -3614,6 +3643,15 @@ packages: whatwg-url: 5.0.0 dev: true + /node-fetch/3.3.0: + resolution: {integrity: sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.0 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + dev: true + /node-int64/0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} dev: true @@ -4965,6 +5003,11 @@ packages: defaults: 1.0.4 dev: true + /web-streams-polyfill/3.2.1: + resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} + engines: {node: '>= 8'} + dev: true + /webidl-conversions/3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} dev: true @@ -5141,3 +5184,7 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} dev: true + + /zod/3.19.1: + resolution: {integrity: sha512-LYjZsEDhCdYET9ikFu6dVPGp2YH9DegXjdJToSzD9rO6fy4qiRYFoyEYwps88OseJlPyl2NOe2iJuhEhL7IpEA==} + dev: true