testing: use Hurl in CI to test Caddy against spec

.github/workflows/ci.yml

@@ -14,6 +14,9 @@ on:
 
 jobs:
   test:
+    permissions:
+      checks: write
+      pull-requests: write
     strategy:
       # Default is true, cancels jobs for other platforms in the matrix if one fails
       fail-fast: false
@@ -140,6 +143,110 @@ jobs:
     #     echo "step_test ${{ steps.step_test.outputs.status }}\n"
     #     exit 1
 
+  spec-test:
+    permissions:
+      checks: write
+      pull-requests: write
+    strategy:
+      matrix:
+        os: 
+          - linux
+        go: 
+          - '1.23'
+
+        include:
+        # Set the minimum Go patch version for the given Go minor
+        # Usable via ${{ matrix.GO_SEMVER }}
+        - go: '1.23'
+          GO_SEMVER: '~1.23.0'
+
+        # Set some variables per OS, usable via ${{ matrix.VAR }}
+        # OS_LABEL: the VM label from GitHub Actions (see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories)
+        # CADDY_BIN_PATH: the path to the compiled Caddy binary, for artifact publishing
+        # SUCCESS: the typical value for $? per OS (Windows/pwsh returns 'True')
+        - os: linux
+          OS_LABEL: ubuntu-latest
+          CADDY_BIN_PATH: ./cmd/caddy/caddy
+          SUCCESS: 0
+
+    runs-on: ${{ matrix.OS_LABEL }}
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: ${{ matrix.GO_SEMVER }}
+        check-latest: true
+
+    - name: Print Go version and environment
+      id: vars
+      shell: bash
+      run: |
+        printf "curl version: $(curl --version)\n"
+        printf "Using go at: $(which go)\n"
+        printf "Go version: $(go version)\n"
+        printf "\n\nGo environment:\n\n"
+        go env
+        printf "\n\nSystem environment:\n\n"
+        env
+        printf "Git version: $(git version)\n\n"
+        # Calculate the short SHA1 hash of the git commit
+        echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+    - name: Get dependencies
+      run: |
+        go get -v -t -d ./...
+        # mkdir test-results
+    - name: Build Caddy
+      working-directory: ./cmd/caddy
+      env:
+        CGO_ENABLED: 0
+      run: |
+        go build -cover -tags nobadger,nopgx,nomysql -trimpath -ldflags="-w -s" -v
+
+    - name: Install Hurl
+      env:
+        HURL_VERSION: "5.0.1"
+      run: |
+        curl --location --remote-name https://github.com/Orange-OpenSource/hurl/releases/download/${HURL_VERSION}/hurl_${HURL_VERSION}_amd64.deb
+        sudo dpkg -i hurl_${HURL_VERSION}_amd64.deb
+        hurl --version
+
+    - name: Run Caddy
+      run: |
+        ./cmd/caddy/caddy environ
+        mkdir coverdir
+        export GOCOVERDIR=./coverdir
+        ./cmd/caddy/caddy start
+
+    - name: Run tests with Hurl
+      run: |
+        mkdir hurl-report
+        find . -name *.hurl -exec hurl --jobs 1 --variables-file caddytest/spec/hurl_vars.properties --very-verbose --verbose --test --report-junit hurl-report/junit.xml --color {} \;
+
+    - name: Publish Test Results
+      uses: EnricoMi/publish-unit-test-result-action@v2
+      with:
+        files: |
+          hurl-report/junit.xml
+
+    - name: Generate Coverage Data
+      run: |
+        export GOCOVERDIR=./coverdir
+        ./cmd/caddy/caddy stop
+        go tool covdata textfmt -i=coverdir -o hurl-report/caddy_cover_${{ steps.vars.outputs.short_sha }}.txt
+        go tool cover -html hurl-report/caddy_cover_${{ steps.vars.outputs.short_sha }}.txt -o hurl-report/caddy_cover_${{ steps.vars.outputs.short_sha }}.html
+
+
+    - name: Publish Coverage Profile
+      uses: actions/upload-artifact@v4
+      with:
+        path: hurl-report/caddy_cover_${{ steps.vars.outputs.short_sha }}.html
+        compression-level: 0
+
   s390x-test:
     name: test (s390x on IBM Z)
     runs-on: ubuntu-latest

caddytest/spec/http/basicauth/spec.hurl

@@ -0,0 +1,38 @@
+# Configure Caddy
+POST http://localhost:2019/load
+Content-Type: text/caddyfile
+```
+{
+	skip_install_trust
+	http_port     9080
+	https_port    9443
+	local_certs
+	debug
+}
+localhost {
+	log
+	basic_auth {
+		john $2a$14$x4HlYwA9Zeer4RkMEYbUzug9XxWmncneR.dcMs.UjalR95URnHg5.
+	}
+	respond "Hello, World!"
+}
+```
+
+# requests without `Authorization` header are rejected with 401
+GET https://localhost:9443
+[Options]
+insecure: true
+HTTP 401
+[Asserts]
+header "WWW-Authenticate" == "Basic realm=\"restricted\""
+
+
+# requests with `Authorization` header are accepted with 200
+GET https://localhost:9443
+[BasicAuth]
+john:password
+[Options]
+insecure: true
+HTTP 200
+[Asserts]
+`Hello, World!`

caddytest/spec/http/file_server/assets/indexed/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>Index.html Title</title>
+	</head>
+	<body>
+		Index.html
+	</body>
+</html>

caddytest/spec/http/file_server/assets/indexed/index.txt

@@ -0,0 +1 @@
+index.txt

caddytest/spec/http/file_server/spec.hurl

@@ -0,0 +1,119 @@
+# Configure Caddy with default configuration
+POST http://localhost:2019/load
+Content-Type: text/caddyfile
+```
+{
+	skip_install_trust
+	http_port     9080
+	https_port    9443
+	local_certs
+	debug
+}
+localhost {
+	root {{indexed_root}}
+	file_server
+}
+```
+
+# requests without specific file receive index file per
+# the default index list: index.html, index.txt
+GET https://localhost:9443
+[Options]
+insecure: true
+HTTP 200
+[Asserts]
+```
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>Index.html Title</title>
+	</head>
+	<body>
+		Index.html
+	</body>
+</html>```
+
+
+# if index.txt is specifically requested, we expect index.txt
+GET https://localhost:9443/index.txt
+[Options]
+insecure: true
+HTTP 200
+[Asserts]
+body == "index.txt"
+
+# requests for sub-folder followed by .. result in sanitized path
+GET https://localhost:9443/non-existent/../index.txt
+[Options]
+insecure: true
+HTTP 200
+[Asserts]
+body == "index.txt"
+
+# results out of root folder are sanitized,
+# and conform to default index list sequence.
+GET https://localhost:9443/../
+[Options]
+insecure: true
+HTTP 200
+[Asserts]
+```
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>Index.html Title</title>
+	</head>
+	<body>
+		Index.html
+	</body>
+</html>```
+
+
+# Configure Caddy with custsom index "index.txt"
+POST http://localhost:2019/load
+Content-Type: text/caddyfile
+```
+{
+	skip_install_trust
+	http_port     9080
+	https_port    9443
+	local_certs
+	debug
+}
+localhost {
+	root {{indexed_root}}
+	file_server {
+		index index.txt
+	}
+}
+```
+
+GET https://localhost:9443
+[Options]
+insecure: true
+HTTP 200
+[Asserts]
+body == "index.txt"
+
+
+# Configure with a root not containing index files
+POST http://localhost:2019/load
+Content-Type: text/caddyfile
+```
+{
+	skip_install_trust
+	http_port     9080
+	https_port    9443
+	local_certs
+	debug
+}
+localhost {
+	root {{unindexed_root}}
+	file_server
+}
+```
+
+GET https://localhost:9443
+[Options]
+insecure: true
+HTTP 404

caddytest/spec/http/headers/spec.hurl

@@ -0,0 +1,22 @@
+# Configure Caddy
+POST http://localhost:2019/load
+Content-Type: text/caddyfile
+```
+{
+	skip_install_trust
+	http_port     9080
+	https_port    9443
+	local_certs
+	debug
+}
+localhost {
+	header "X-Custom-Header" "Custom-Value"
+}
+```
+
+GET https://localhost:9443
+[Options]
+insecure: true
+HTTP 200
+[Asserts]
+header "X-Custom-Header" == "Custom-Value"

caddytest/spec/http/requestbody/spec.hurl

@@ -0,0 +1,36 @@
+# Configure Caddy
+POST http://localhost:2019/load
+Content-Type: text/caddyfile
+```
+{
+	skip_install_trust
+	http_port     9080
+	https_port    9443
+	local_certs
+}
+localhost {
+	log
+	request_body {
+        max_size 2B
+    }
+    reverse_proxy localhost:8000 # to fake body reading
+	handle_errors 4xx {
+		respond "OK"
+	}
+}
+http://localhost:8000 {
+	respond "Failed"
+}
+```
+
+GET https://localhost:9443
+[Options]
+delay: 1s
+insecure: true
+```
+Hello
+```
+HTTP 413
+`OK`
+
+# TODO: how to test{read,write}_timeout?