From 91f9705ead99d6a6b4a2f1cf8435e4b64a3a0f2e Mon Sep 17 00:00:00 2001 From: Cedric Bail Date: Mon, 8 Jan 2024 16:23:25 -0700 Subject: [PATCH 1/8] Add a github action that run the benchmark and report it as comment to the PR. --- .github/workflows/bench.yml | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/bench.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 00000000..4effc036 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,72 @@ +name: Run benchmark + +# Cancel the workflow in progress in newer build is about to start. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + pull_request: + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: "1.21" + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run benchmark on commit + run: go test -bench=. -benchmem -count=20 -run=^# > pr.bench + + - name: Switch to master branch + run: git checkout master + - name: Run benchmark on master + id: benchmark-master + run: go test -bench=. -benchmem -count=20 -run=^# > master.bench + + - name: Restore benchstat + id: cache-benchstat + uses: actions/cache@v3 + with: + path: ~/go/bin/benchstat + key: ${{ runner.os }}-benchstat + - name: Install benchstat + if: steps.cache-benchstat.outputs.cache-hit != 'true' + run: go install golang.org/x/perf/cmd/benchstat@latest + - name: Save benchstat + if: steps.cache-benchstat.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: ~/go/bin/benchstat + key: ${{ steps.cache-benchstat.outputs.cache-primary-key }} + + - name: Compare benchmarks + id: compare + run: | + echo "STAT<> "$GITHUB_OUTPUT" + benchstat pr.bench master.bench >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + - name: Comment benchmark result + if: ${{ !env.ACT }} + continue-on-error: true + uses: marocchino/sticky-pull-request-comment@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + header: bench + message: | + ### Benchmark Result +
Benchmark result compared against master branch + + ``` + ${{ steps.compare.outputs.STAT }} + ``` +
+ - name: Display result on command line when using act + if: ${{ env.ACT }} + run: | + echo "Benchmark result compared against master branch" + echo "${{ steps.compare.outputs.STAT }}" From 0bfaff0d07dbd8562d36ce1f0e43c65831877aed Mon Sep 17 00:00:00 2001 From: Cedric Bail Date: Mon, 8 Jan 2024 16:24:21 -0700 Subject: [PATCH 2/8] Cancel in progress job When running test and linter, there is no reason to keep going when a new head is pushed. So cancel past build to enable new build to start asap. --- .github/workflows/go.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0bf70008..838e495b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -1,5 +1,10 @@ name: Run Tests +# Cancel the workflow in progress in newer build is about to start. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + on: push: branches: @@ -65,3 +70,4 @@ jobs: uses: codecov/codecov-action@v4 with: flags: ${{ matrix.os }},go-${{ matrix.go }} + \ No newline at end of file From ee6135df13275070aec101da40cc6f7657e20f13 Mon Sep 17 00:00:00 2001 From: Cedric Bail Date: Mon, 8 Jan 2024 17:27:42 -0700 Subject: [PATCH 3/8] Use only actions/checkout and do parallel benchmark. --- .github/workflows/bench.yml | 40 ++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 4effc036..15ad4e5e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -9,8 +9,10 @@ on: pull_request: jobs: - benchmark: + benchmark-pr: runs-on: ubuntu-latest + outputs: + result: ${{ steps.benchmark.outputs.result }} steps: - name: Setup go uses: actions/setup-go@v5 @@ -20,14 +22,40 @@ jobs: uses: actions/checkout@v4 - name: Run benchmark on commit - run: go test -bench=. -benchmem -count=20 -run=^# > pr.bench + id: benchmark + run: | + echo "result<> "$GITHUB_OUTPUT" + go test -bench=. -benchmem -count=20 -run=^# >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + benchmark-master: + runs-on: ubuntu-latest + outputs: + result: ${{ steps.benchmark.outputs.result }} + steps: + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: "1.21" - name: Switch to master branch - run: git checkout master + uses: actions/checkout@v4 + with: + ref: master - name: Run benchmark on master - id: benchmark-master - run: go test -bench=. -benchmem -count=20 -run=^# > master.bench + id: benchmark + run: | + echo "result<> "$GITHUB_OUTPUT" + go test -bench=. -benchmem -count=20 -run=^# >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + benchmark: + runs-on: ubuntu-latest + needs: [benchmark-pr, benchmark-master] + steps: + - name: Setup go + uses: actions/setup-go@v5 + with: + go-version: "1.21" - name: Restore benchstat id: cache-benchstat uses: actions/cache@v3 @@ -47,6 +75,8 @@ jobs: - name: Compare benchmarks id: compare run: | + echo "${{ needs.benchmark-pr.outputs.result }}" > pr.bench + echo "${{ needs.benchmark-master.outputs.result }}" > master.bench echo "STAT<> "$GITHUB_OUTPUT" benchstat pr.bench master.bench >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" From 32d8b13e4f1524c2addb80b2b0d2d17bdb239ff4 Mon Sep 17 00:00:00 2001 From: Cedric Bail Date: Mon, 8 Jan 2024 17:36:45 -0700 Subject: [PATCH 4/8] Need write access to PR to add/edit comments actually. --- .github/workflows/bench.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 15ad4e5e..66f83f05 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -8,6 +8,9 @@ concurrency: on: pull_request: +permissions: + pull-requests: write + jobs: benchmark-pr: runs-on: ubuntu-latest From a43806976019b883b37ec61be57ee291bc2b1aeb Mon Sep 17 00:00:00 2001 From: Cedric Bail Date: Mon, 8 Jan 2024 17:46:42 -0700 Subject: [PATCH 5/8] Only benchmakr step need write permissions to PR. --- .github/workflows/bench.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 66f83f05..26dd3d8d 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -8,9 +8,6 @@ concurrency: on: pull_request: -permissions: - pull-requests: write - jobs: benchmark-pr: runs-on: ubuntu-latest @@ -54,6 +51,8 @@ jobs: benchmark: runs-on: ubuntu-latest needs: [benchmark-pr, benchmark-master] + permissions: + pull-requests: write steps: - name: Setup go uses: actions/setup-go@v5 From 92ad5cf6ef46f9f7832101f7ed88e96c11d7d610 Mon Sep 17 00:00:00 2001 From: Cedric Bail Date: Tue, 16 Apr 2024 16:15:46 -0600 Subject: [PATCH 6/8] Benchmark pr against master to show gain/loss in a more readable way. --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 26dd3d8d..c5be65d8 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -80,7 +80,7 @@ jobs: echo "${{ needs.benchmark-pr.outputs.result }}" > pr.bench echo "${{ needs.benchmark-master.outputs.result }}" > master.bench echo "STAT<> "$GITHUB_OUTPUT" - benchstat pr.bench master.bench >> "$GITHUB_OUTPUT" + benchstat master.bench pr.bench >> "$GITHUB_OUTPUT" echo "EOF" >> "$GITHUB_OUTPUT" - name: Comment benchmark result if: ${{ !env.ACT }} From 1b307d84ea16156d938a9c47bd2a51c76b35cd25 Mon Sep 17 00:00:00 2001 From: Cedric Bail Date: Tue, 16 Apr 2024 16:28:41 -0600 Subject: [PATCH 7/8] Add permission to write to comments. --- .github/workflows/bench.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c5be65d8..c7fdbd84 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -8,6 +8,10 @@ concurrency: on: pull_request: +permissions: + contents: read + pull-requests: write + jobs: benchmark-pr: runs-on: ubuntu-latest From 60bff745a82c005fbbf1a3da8607a545ebf15f26 Mon Sep 17 00:00:00 2001 From: Cedric Bail Date: Tue, 16 Apr 2024 16:38:42 -0600 Subject: [PATCH 8/8] No need to be explicit about the GITHUB_TOKEN and hide previous result. --- .github/workflows/bench.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c7fdbd84..210bd253 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -91,8 +91,9 @@ jobs: continue-on-error: true uses: marocchino/sticky-pull-request-comment@v2 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} header: bench + hide: true + hide_classify: "OUTDATED" message: | ### Benchmark Result
Benchmark result compared against master branch