Continuously benchmark fnm (#877)

* wip

* install hyperfine

* switch sides

* add permissions block

* permissions

* use ternary

* wip wip wip

* wip

* other way?

* force store

* simplify

* add more perms

* add file size and units

* change columns

* add header

* add hash

* reverse the conditional to store

* show last value for 0

* lastValue is now being sent

* 0

* the return of all the build

* Add min runs

.github/workflows/rust.yml

@@ -106,7 +106,7 @@ jobs:
         run_install: false
     - uses: actions/setup-node@v3
       with:
-        node-version: 16.x
+        node-version: 18.x
         cache: 'pnpm'
     - name: Get pnpm store directory
       id: pnpm-cache
@@ -140,7 +140,7 @@ jobs:
         run_install: false
     - uses: actions/setup-node@v3
       with:
-        node-version: 16.x
+        node-version: 18.x
         cache: 'pnpm'
     - name: Get pnpm store directory
       id: pnpm-cache
@@ -176,7 +176,7 @@ jobs:
   #       run_install: false
   #   - uses: actions/setup-node@v3
   #     with:
-  #       node-version: 16.x
+  #       node-version: 18.x
   #       cache: 'pnpm'
   #   - name: Get pnpm store directory
   #     id: pnpm-cache
@@ -212,7 +212,7 @@ jobs:
         run_install: false
     - uses: actions/setup-node@v3
       with:
-        node-version: 16.x
+        node-version: 18.x
         cache: 'pnpm'
     - name: Get pnpm store directory
       id: pnpm-cache
@@ -340,7 +340,7 @@ jobs:
         run_install: false
     - uses: actions/setup-node@v3
       with:
-        node-version: 16.x
+        node-version: 18.x
         cache: 'pnpm'
     - name: Get pnpm store directory
       id: pnpm-cache
@@ -357,3 +357,72 @@ jobs:
     - name: Generate command markdown
       run: |
         pnpm run generate-command-docs --check --binary-path=$(which fnm)
+
+  run_e2e_benchmarks:
+    runs-on: ubuntu-latest
+    name: bench/linux
+    needs: [build_static_linux_binary]
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+    - name: install necessary shells
+      run: sudo apt-get update && sudo apt-get install -y fish zsh bash hyperfine
+    - uses: actions/checkout@v3
+    - uses: actions/download-artifact@v3
+      with:
+        name: fnm-linux
+        path: target/release
+    - name: mark binary as executable
+      run: chmod +x target/release/fnm
+    - name: install fnm as binary
+      run: |
+        sudo install target/release/fnm /bin
+        fnm --version
+    - uses: pnpm/action-setup@v2.2.4
+      with:
+        run_install: false
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18.x
+        cache: 'pnpm'
+    - name: Get pnpm store directory
+      id: pnpm-cache
+      run: |
+        echo "::set-output name=pnpm_cache_dir::$(pnpm store path)"
+    - uses: actions/cache@v3
+      name: Setup pnpm cache
+      with:
+        path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+        restore-keys: |
+          ${{ runner.os }}-pnpm-store-
+    - run: pnpm install
+    - name: Run benchmarks
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        SHOULD_STORE: ${{ toJson(!github.event.pull_request) }}
+      id: benchmark
+      run: |
+        delimiter="$(openssl rand -hex 8)"
+        echo "markdown<<${delimiter}" >> "${GITHUB_OUTPUT}"
+        node benchmarks/run.mjs --store=$SHOULD_STORE >> "${GITHUB_OUTPUT}"
+        echo "${delimiter}" >> "${GITHUB_OUTPUT}"
+
+    - name: Create a PR comment
+      if: ${{ github.event.pull_request }}
+      uses: thollander/actions-comment-pull-request@v1
+      with:
+        message: |
+          <!--benchy comment-->
+          ## Linux Benchmarks for ${{ github.event.pull_request.head.sha }}
+          ${{ steps.benchmark.outputs.markdown }}
+        comment_includes: '<!--benchy comment-->'
+
+    - name: Create a commit comment
+      if: ${{ !github.event.pull_request }}
+      uses: peter-evans/commit-comment@v2
+      with:
+        body: |
+          ## Linux Benchmarks
+          ${{ steps.benchmark.outputs.markdown }}

benchmarks/basic/fnm

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+eval "$(fnm env --multi)"
+fnm install v10.11.0
+fnm use v10.11.0
+node -v

benchmarks/basic/fnm_latest_master

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+eval "$(~/.fnm-latest/fnm env --multi)"
+~/.fnm-latest/fnm install v10.11.0
+~/.fnm-latest/fnm use v10.11.0
+node -v

benchmarks/basic/fnm_reason

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+eval "$(~/.fnm/fnm env --multi)"
+~/.fnm/fnm install v10.11.0
+~/.fnm/fnm use v10.11.0
+node -v

benchmarks/basic/nvm

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export NVM_DIR="$HOME/.nvm"
+\. "$NVM_DIR/nvm.sh"  # This loads nvm
+
+nvm install v10.11.0
+nvm use v10.11.0
+node -v

benchmarks/run

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+export FNM_DIR
+
+BASE_DIR="$(dirname "$(realpath "$0")")"
+cd "$BASE_DIR" || exit 1
+
+FNM_DIR="$(mktemp -d)"
+export PATH="$BASE_DIR/../target/release:$PATH"
+
+mkdir results 2>/dev/null || :
+
+if [ ! -f "$BASE_DIR/../target/release/fnm" ]; then
+  echo "Can't access the release version of fnm.rs"
+  exit 1
+fi
+
+if ! command -v hyperfine >/dev/null 2>&1; then
+  echo "Can't access Hyperfine. Are you sure it is installed?"
+  echo "  if not, visit https://github.com/sharkdp/hyperfine"
+  exit 1
+fi
+
+# Running it with warmup means we're going to have the versions
+# pre-installed. I think it is good because you open your shell more times
+# than you install Node versions.
+hyperfine \
+  --warmup=2 \
+  --min-runs=40 \
+  --time-unit=millisecond \
+  --export-json="./results/basic.json" \
+  --export-markdown="./results/basic.md" \
+  "basic/nvm" \
+  "basic/fnm"

benchmarks/run.mjs

@@ -0,0 +1,272 @@
+// @ts-check
+
+import z from "zod"
+import os from "node:os"
+import path from "node:path"
+import fetch from "node-fetch"
+import { execa } from "execa"
+import { binary, command, flag, option } from "cmd-ts"
+import Url from "cmd-ts/dist/cjs/batteries/url.js"
+import { run } from "cmd-ts"
+import fs from "node:fs/promises"
+import { dedent } from "ts-dedent"
+
+const HyperfineResult = z.object({
+  results: z.array(
+    z.object({
+      command: z.string(),
+      mean: z.number(),
+      stddev: z.number(),
+      median: z.number(),
+      user: z.number(),
+      system: z.number(),
+      min: z.number(),
+      max: z.number(),
+      times: z.array(z.number()),
+      exit_codes: z.array(z.literal(0)),
+    })
+  ),
+})
+
+const BenchyResult = z.object({
+  data: z.object({
+    embed: z.object({
+      small: z.string(),
+      big: z.string(),
+
+      currentValue: z.number(),
+      lastValue: z.number().optional(),
+      diff: z
+        .object({
+          value: z.number(),
+          arrowImage: z.string(),
+        })
+        .optional(),
+    }),
+  }),
+})
+
+const { HttpUrl } = Url
+
+const cmd = command({
+  name: "run-benchmarks",
+  args: {
+    serverUrl: option({
+      long: "server-url",
+      type: HttpUrl,
+      defaultValue: () => new URL("https://benchy.hagever.com"),
+      defaultValueIsSerializable: true,
+    }),
+    githubToken: option({
+      long: "github-token",
+      env: "GITHUB_TOKEN",
+    }),
+    shouldStore: flag({
+      long: "store",
+    }),
+  },
+  async handler({ serverUrl, githubToken, shouldStore }) {
+    const repoName = "fnm"
+    const repoOwner = "schniz"
+
+    const hyperfineResult = await runHyperfine()
+
+    if (!hyperfineResult.success) {
+      console.error(
+        `Can't run benchmarks: wrong data:`,
+        hyperfineResult.error.issues
+      )
+      process.exitCode = 1
+      return
+    }
+
+    const { results } = hyperfineResult.data
+
+    const url = new URL("/api/metrics", serverUrl)
+    const trackedKeys = ["median", "max", "mean", "min", "stddev"]
+
+    const metrics = results
+      .flatMap((result) => {
+        return trackedKeys.map((key) => {
+          return {
+            displayName: `${result.command}/${key}`,
+            value: result[key] * 1000, // everything is in seconds
+            units: "ms",
+          }
+        })
+      })
+      .concat([
+        {
+          displayName: `binary size`,
+          value: await getFilesize(),
+          units: "kb",
+        },
+      ])
+      .map((metric) => {
+        return {
+          ...metric,
+          key: `${os.platform()}/${os.arch()}/${metric.displayName}`,
+        }
+      })
+
+    const embeds$ = metrics.map(async ({ key, value, displayName, units }) => {
+      const response = await fetch(String(url), {
+        method: "PUT",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          coloring: "lower-is-better",
+          repoOwner,
+          repoName,
+          githubToken,
+          key,
+          value,
+        }),
+      })
+
+      if (!response.ok) {
+        throw new Error(`Response is not okay: ${response.status}`)
+      }
+
+      const { data } = BenchyResult.parse(await response.json())
+      return {
+        displayName,
+        units,
+        ...data.embed,
+      }
+    })
+
+    const embeds = await Promise.all(embeds$)
+
+    const table = (() => {
+      const rows = embeds
+        .map((data) => {
+          return dedent`
+            <tr>
+              <td><code>${data.displayName}</code></td>
+              <td><code>${round(data.currentValue, 2)}${data.units}</code></td>
+              <td>${
+                typeof data.lastValue === "undefined"
+                  ? ""
+                  : `<code>${round(data.lastValue, 2)}${data.units}</code>`
+              }</td>
+              <td>${
+                !data.diff
+                  ? "<code>0</code>"
+                  : dedent`
+                  <picture title=${JSON.stringify(
+                    data.diff.value > 0 ? "increase" : "decrease"
+                  )}>
+                    <img width="16" valign="middle" src="${
+                      data.diff.arrowImage
+                    }">
+                  </picture>
+                  <code>${data.diff.value > 0 ? "+" : ""}${round(
+                      data.diff.value,
+                      2
+                    )}${data.units}</code>
+                    `
+              }</td>
+              <td>
+                <details><summary><img valign="middle" src="${
+                  data.small
+                }" /></summary><br/><img src="${data.big}" /></details>
+              </td>
+            </tr>
+          `
+        })
+        .join("\n")
+      return dedent`
+        <table>
+          <thead>
+            <tr>
+              <th align="left">benchmark</th>
+              <th>current value</th>
+              <th>last value</th>
+              <th>diff</th>
+              <th>trend</th>
+            </tr>
+          </thead>
+          <tbody>
+            ${rows}
+          </tbody>
+        </table>
+      `
+    })()
+
+    console.log(table)
+
+    if (shouldStore) {
+      const response = await fetch(String(url), {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          repoOwner,
+          repoName,
+          githubToken,
+          metrics,
+        }),
+      })
+
+      if (!response.ok) {
+        throw new Error(`Response is not okay: ${response.status}`)
+      }
+
+      console.error(await response.json())
+    }
+  },
+})
+
+/**
+ * @param {number} number
+ * @param {number} digits
+ */
+function round(number, digits) {
+  const pow = Math.pow(10, digits)
+  return Math.round(number * pow) / pow
+}
+
+/**
+ * Returns the size of the `fnm` binary in kilobytes
+ *
+ * @returns number
+ */
+async function getFilesize() {
+  const fnmBinary = await execa("which", ["fnm"])
+  const stat = await fs.stat(fnmBinary.stdout.trim())
+  return Math.round(stat.size / 1024)
+}
+
+async function runHyperfine() {
+  const file = path.join(os.tmpdir(), `bench-${Date.now()}.json`)
+  await execa(
+    `hyperfine`,
+    [
+      "--min-runs=20",
+      `--export-json=${file}`,
+      "--warmup=2",
+      ...[
+        "--command-name=fnm_basic",
+        new URL("./basic/fnm", import.meta.url).pathname,
+      ],
+      // ...[
+      //   "--command-name=nvm_basic",
+      //   new URL("./basic/nvm", import.meta.url).pathname,
+      // ],
+    ],
+    {
+      stdout: process.stderr,
+      stderr: process.stderr,
+      stdin: "ignore",
+    }
+  )
+
+  const json = JSON.parse(await fs.readFile(file, "utf8"))
+  const parsed = HyperfineResult.safeParse(json)
+  return parsed
+}
+
+run(binary(cmd), process.argv)

package.json

@@ -33,6 +33,7 @@
     "execa": "6.1.0",
     "jest": "^29.3.1",
     "lerna-changelog": "2.2.0",
+    "node-fetch": "^3.3.0",
     "prettier": "2.8.1",
     "pv": "1.0.1",
     "shell-escape": "^0.2.0",
@@ -40,7 +41,8 @@
     "toml": "3.0.0",
     "ts-dedent": "^2.2.0",
     "ts-jest": "^29.0.3",
-    "typescript": "^4.8.4"
+    "typescript": "^4.8.4",
+    "zod": "^3.19.1"
   },
   "prettier": {
     "semi": false

pnpm-lock.yaml

@@ -12,6 +12,7 @@ specifiers:
   execa: 6.1.0
   jest: ^29.3.1
   lerna-changelog: 2.2.0
+  node-fetch: ^3.3.0
   prettier: 2.8.1
   pv: 1.0.1
   shell-escape: ^0.2.0
@@ -20,6 +21,7 @@ specifiers:
   ts-dedent: ^2.2.0
   ts-jest: ^29.0.3
   typescript: ^4.8.4
+  zod: ^3.19.1
 
 devDependencies:
   '@changesets/cli': 2.25.2
@@ -33,6 +35,7 @@ devDependencies:
   execa: 6.1.0
   jest: 29.3.1_@types+node@18.11.9
   lerna-changelog: 2.2.0
+  node-fetch: 3.3.0
   prettier: 2.8.1
   pv: 1.0.1
   shell-escape: 0.2.0
@@ -41,6 +44,7 @@ devDependencies:
   ts-dedent: 2.2.0
   ts-jest: 29.0.3_4f6uxrzmuwipl5rr3bcogf6k74
   typescript: 4.9.3
+  zod: 3.19.1
 
 packages:
 
@@ -1842,6 +1846,11 @@ packages:
       array-find-index: 1.0.2
     dev: true
 
+  /data-uri-to-buffer/4.0.0:
+    resolution: {integrity: sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==}
+    engines: {node: '>= 12'}
+    dev: true
+
   /dataloader/1.4.0:
     resolution: {integrity: sha512-68s5jYdlvasItOJnCuI2Q9s4q98g0pCyL3HrcKJu8KNugUl8ahgmZYg38ysLTgQjjXX3H8CJLkAvWrclWfcalw==}
     dev: true
@@ -2173,6 +2182,14 @@ packages:
       bser: 2.1.1
     dev: true
 
+  /fetch-blob/3.2.0:
+    resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
+    engines: {node: ^12.20 || >= 14.13}
+    dependencies:
+      node-domexception: 1.0.0
+      web-streams-polyfill: 3.2.1
+    dev: true
+
   /fill-range/7.0.1:
     resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==}
     engines: {node: '>=8'}
@@ -2211,6 +2228,13 @@ packages:
       pkg-dir: 4.2.0
     dev: true
 
+  /formdata-polyfill/4.0.10:
+    resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
+    engines: {node: '>=12.20.0'}
+    dependencies:
+      fetch-blob: 3.2.0
+    dev: true
+
   /fs-extra/7.0.1:
     resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==}
     engines: {node: '>=6 <7 || >=8'}
@@ -3595,6 +3619,11 @@ packages:
     engines: {node: '>= 0.6'}
     dev: true
 
+  /node-domexception/1.0.0:
+    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
+    engines: {node: '>=10.5.0'}
+    dev: true
+
   /node-fetch/1.7.3:
     resolution: {integrity: sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==}
     dependencies:
@@ -3614,6 +3643,15 @@ packages:
       whatwg-url: 5.0.0
     dev: true
 
+  /node-fetch/3.3.0:
+    resolution: {integrity: sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      data-uri-to-buffer: 4.0.0
+      fetch-blob: 3.2.0
+      formdata-polyfill: 4.0.10
+    dev: true
+
   /node-int64/0.4.0:
     resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==}
     dev: true
@@ -4965,6 +5003,11 @@ packages:
       defaults: 1.0.4
     dev: true
 
+  /web-streams-polyfill/3.2.1:
+    resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
+    engines: {node: '>= 8'}
+    dev: true
+
   /webidl-conversions/3.0.1:
     resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
     dev: true
@@ -5141,3 +5184,7 @@ packages:
     resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
     engines: {node: '>=10'}
     dev: true
+
+  /zod/3.19.1:
+    resolution: {integrity: sha512-LYjZsEDhCdYET9ikFu6dVPGp2YH9DegXjdJToSzD9rO6fy4qiRYFoyEYwps88OseJlPyl2NOe2iJuhEhL7IpEA==}
+    dev: true