storybook-chromatic.yml

name: Storybook

on:
    pull_request:
        paths: # Only run if the frontend has changed
            - 'frontend/**'
            - '.storybook/**'
            - 'package.json'
            - '.github/workflows/storybook-chromatic.yml'
            - 'playwright.config.ts'

concurrency:
    group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
    # This is so that the workflow run isn't canceled when a snapshot update is pushed within it by posthog-bot
    # We do however cancel from container-images-ci.yml if a commit is pushed by someone OTHER than posthog-bot
    cancel-in-progress: false

jobs:
    storybook-chromatic:
        name: Publish to Chromatic
        runs-on: ubuntu-latest
        timeout-minutes: 15
        if: github.event.pull_request.head.repo.full_name == github.repository # Don't run on forks
        outputs:
            storybook-url: ${{ steps.publish.outputs.storybookUrl }}
        steps:
            - uses: actions/checkout@v3
              with:
                  fetch-depth: 0 # 👈 Required to retrieve git history (https://www.chromatic.com/docs/github-actions)

            - name: Install pnpm
              uses: pnpm/action-setup@v4

            - name: Set up Node.js
              uses: actions/setup-node@v4
              with:
                  node-version: 18.12.1
                  cache: pnpm

            - name: Install dependencies and Chromatic
              run: pnpm i -D chromatic

            - name: Publish to Chromatic
              uses: chromaui/action@v11
              id: publish
              with:
                  token: ${{ secrets.GITHUB_TOKEN }}
                  # 👇 Chromatic projectToken, refer to the manage page to obtain it.
                  projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}

    visual-regression:
        name: Visual regression tests
        runs-on: ubuntu-latest
        timeout-minutes: 30
        container:
            image: mcr.microsoft.com/playwright:v1.45.0
        strategy:
            fail-fast: false
            matrix:
                browser: ['chromium', 'webkit']
                shard: [1, 2]
        env:
            SHARD_COUNT: '2'
            CYPRESS_INSTALL_BINARY: '0'
            NODE_OPTIONS: --max-old-space-size=6144
            OPT_OUT_CAPTURE: 1
        outputs:
            # The below have to be manually listed unfortunately, as GitHub Actions doesn't allow matrix-dependent outputs
            chromium-1-added: ${{ steps.diff.outputs.chromium-1-added }}
            chromium-1-modified: ${{ steps.diff.outputs.chromium-1-modified }}
            chromium-1-deleted: ${{ steps.diff.outputs.chromium-1-deleted }}
            chromium-1-total: ${{ steps.diff.outputs.chromium-1-total }}
            chromium-1-commitHash: ${{ steps.commit-hash.outputs.chromium-1-commitHash }}
            chromium-2-added: ${{ steps.diff.outputs.chromium-2-added }}
            chromium-2-modified: ${{ steps.diff.outputs.chromium-2-modified }}
            chromium-2-deleted: ${{ steps.diff.outputs.chromium-2-deleted }}
            chromium-2-total: ${{ steps.diff.outputs.chromium-2-total }}
            chromium-2-commitHash: ${{ steps.commit-hash.outputs.chromium-2-commitHash }}
            webkit-1-added: ${{ steps.diff.outputs.webkit-1-added }}
            webkit-1-modified: ${{ steps.diff.outputs.webkit-1-modified }}
            webkit-1-deleted: ${{ steps.diff.outputs.webkit-1-deleted }}
            webkit-1-total: ${{ steps.diff.outputs.webkit-1-total }}
            webkit-1-commitHash: ${{ steps.commit-hash.outputs.webkit-1-commitHash }}
            webkit-2-added: ${{ steps.diff.outputs.webkit-2-added }}
            webkit-2-modified: ${{ steps.diff.outputs.webkit-2-modified }}
            webkit-2-deleted: ${{ steps.diff.outputs.webkit-2-deleted }}
            webkit-2-total: ${{ steps.diff.outputs.webkit-2-total }}
            webkit-2-commitHash: ${{ steps.commit-hash.outputs.webkit-2-commitHash }}
            firefox-1-added: ${{ steps.diff.outputs.firefox-1-added }}
            firefox-1-modified: ${{ steps.diff.outputs.firefox-1-modified }}
            firefox-1-deleted: ${{ steps.diff.outputs.firefox-1-deleted }}
            firefox-1-total: ${{ steps.diff.outputs.firefox-1-total }}
            firefox-1-commitHash: ${{ steps.commit-hash.outputs.firefox-1-commitHash }}
            firefox-2-added: ${{ steps.diff.outputs.firefox-2-added }}
            firefox-2-modified: ${{ steps.diff.outputs.firefox-2-modified }}
            firefox-2-deleted: ${{ steps.diff.outputs.firefox-2-deleted }}
            firefox-2-total: ${{ steps.diff.outputs.firefox-2-total }}
            firefox-2-commitHash: ${{ steps.commit-hash.outputs.firefox-2-commitHash }}
        steps:
            - uses: actions/checkout@v3
              with:
                  fetch-depth: 1
                  repository: ${{ github.event.pull_request.head.repo.full_name }}
                  ref: ${{ github.event.pull_request.head.ref }}
                  # Use PostHog Bot token when not on forks to enable proper snapshot updating
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}

            - name: Install pnpm
              uses: pnpm/action-setup@v4

            - name: Set up Node.js
              uses: buildjet/setup-node@v3
              with:
                  node-version: 18.12.1
                  cache: pnpm

            - name: Install package.json dependencies with pnpm
              run: pnpm install --frozen-lockfile

            - name: Install CI utilities with pnpm
              run: pnpm install http-server wait-on

            - name: Build Storybook
              run: pnpm build-storybook --test --quiet # Silence since progress logging results in a massive wall of spam

            - name: Serve Storybook in the background
              run: |
                  retries=3
                  while [ $retries -gt 0 ]; do
                      pnpm exec http-server storybook-static --port 6006 --silent &
                      if pnpm wait-on http://127.0.0.1:6006 --timeout 15; then
                          break
                      fi
                      retries=$((retries-1))
                      echo "Failed to serve Storybook, retrying... ($retries retries left)"
                  done

            - name: Run @storybook/test-runner
              env:
                  # Solving this bug by overriding $HOME: https://github.com/microsoft/playwright/issues/6500
                  HOME: /root
                  # Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
                  VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
                  STORYBOOK_SKIP_TAGS: 'test-skip,test-skip-${{ matrix.browser }}'
              run: |
                  pnpm test:visual:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT

            - name: Archive failure screenshots
              if: ${{ failure() }}
              uses: actions/upload-artifact@v3
              with:
                  name: failure-screenshots-${{ matrix.browser }}
                  path: frontend/__snapshots__/__failures__/

            - name: Count and optimize updated snapshots
              id: diff
              # Skip on forks
              if: github.event.pull_request.head.repo.full_name == github.repository
              run: |
                  git config --global --add safe.directory '*' # Calm git down about file ownership
                  git diff --name-status frontend/__snapshots__/ # For debugging
                  ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l)
                  MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l)
                  DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l)
                  TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l)

                  # If added or modified, run OptiPNG
                  if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then
                    echo "Snapshots updated ($ADDED new, $MODIFIED changed), running OptiPNG"
                    apt update && apt install -y optipng
                    optipng -clobber -o4 -strip all

                    # we don't want to _always_ run OptiPNG
                    # so, we run it after checking for a diff
                    # but, the files we diffed might then be changed by OptiPNG
                    # and as a result they might no longer be different...

                    # we check again
                    git diff --name-status frontend/__snapshots__/ # For debugging
                    ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l)
                    MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l)
                    DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l)
                    TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l)

                    if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then
                        echo "Snapshots updated ($ADDED new, $MODIFIED changed), _even after_ running OptiPNG"
                        git add frontend/__snapshots__/ playwright/
                    fi
                  fi

                  echo "${{ matrix.browser }}-${{ matrix.shard }}-added=$ADDED" >> $GITHUB_OUTPUT
                  echo "${{ matrix.browser }}-${{ matrix.shard }}-modified=$MODIFIED" >> $GITHUB_OUTPUT
                  echo "${{ matrix.browser }}-${{ matrix.shard }}-deleted=$DELETED" >> $GITHUB_OUTPUT
                  echo "${{ matrix.browser }}-${{ matrix.shard }}-total=$TOTAL" >> $GITHUB_OUTPUT

            - name: Commit updated snapshots
              uses: EndBug/add-and-commit@v9
              if: github.event.pull_request.head.repo.full_name == github.repository
              id: commit
              with:
                  add: '["frontend/__snapshots__/", "playwright/"]'
                  message: 'Update UI snapshots for `${{ matrix.browser }}` (${{ matrix.shard }})'
                  pull: --rebase --autostash # Make sure we're up to date with other browsers' updates
                  default_author: github_actions
                  github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}

            - name: Add commit hash to outputs, including browser name
              id: commit-hash
              if: steps.commit.outputs.pushed == 'true'
              run: echo "${{ matrix.browser }}-${{ matrix.shard }}-commitHash=${{ steps.commit.outputs.commit_long_sha }}" >> $GITHUB_OUTPUT

    visual-regression-summary:
        name: Summarize visual regression tests
        runs-on: ubuntu-latest
        timeout-minutes: 5
        needs: visual-regression
        if: always() # Run even if visual-regression fails for one (or more) of the browsers
        steps:
            - name: Post comment about updated snapshots
              if: github.event.pull_request.head.repo.full_name == github.repository
              uses: actions/github-script@v6
              with:
                  github-token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
                  script: |
                      const BROWSERS = ['chromium', 'webkit']

                      const diffJobOutputs = ${{ toJson(needs.visual-regression.outputs) }}
                      const summaryDiff = { total: 0, added: 0, modified: 0, deleted: 0 }
                      const diffByBrowser = Object.fromEntries(BROWSERS.map(browser => [browser, {
                        total: 0, added: 0, modified: 0, deleted: 0, commitHashes: []
                      }]))
                      for (const [key, rawValue] of Object.entries(diffJobOutputs)) {
                        // Split e.g. 'chromium-1-commitHash' into ['chromium', '1' 'commitHash']
                        const [browser, shardNumber, diffKey] = key.split('-')
                        // Sum up the counts - but not the commit hash
                        if (diffKey === 'commitHash') {
                          diffByBrowser[browser].commitHashes.push([parseInt(shardNumber), rawValue])
                        } else {
                          const value = parseInt(rawValue)
                          diffByBrowser[browser][diffKey] += value
                          summaryDiff[diffKey] += value
                        }
                      }

                      for (const browser of BROWSERS) {
                        if (diffByBrowser[browser]?.total === undefined) {
                          diffByBrowser[browser] = null // Null means failure
                        }
                      }

                      if (summaryDiff.total === 0) {
                        console.log('No changes were made, skipping comment')
                        return
                      }

                      const diffByBrowserDisplay = Object.entries(diffByBrowser).map(([browser, diff]) => {
                        if (!diff) {
                          return `- \`${browser}\`: failed`
                        }
                        const { added: a, modified: m, deleted: d, commitHashes } = diff
                        const b = a + m + d > 0 ? '**' : '' // Bold list item if there were changes
                        let extraInfo = ''
                        if (b) {
                          const commitInfo = commitHashes.map(
                            ([shardNumber, commitHash]) =>
                            `[diff for shard ${shardNumber}](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${commitHash})`
                          ).join(', ') || "wasn't pushed!"
                          extraInfo = ` (${commitInfo})`
                        }
                        return `- ${b}\`${browser}\`${b}: **${a}** added, **${m}** modified, **${d}** deleted${extraInfo}`
                      }).join('\n')

                      github.rest.issues.createComment({
                        issue_number: context.issue.number,
                        owner: context.repo.owner,
                        repo: context.repo.repo,
                        body: `## 📸 UI snapshots have been updated

                      **${summaryDiff.total}** snapshot changes in total. **${summaryDiff.added}** added, **${summaryDiff.modified}** modified, **${summaryDiff.deleted}** deleted:

                      ${diffByBrowserDisplay}

                      Triggered by [this commit](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${{ github.sha }}).

                      👉 **[Review this PR's diff of snapshots.](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/files#:~:text=frontend/__snapshots__/)**`
                      })