name: perfcheck on: issue_comment: types: [created, edited] paths: - .github/workflows/perfcheck.yml - .cargo/config.toml - scripts/ci/vegeta_report_to_csv.jq - Cargo.lock - Cargo.toml - src/** jobs: benchmark: if: github.event.issue.pull_request && contains(github.event.comment.body, '/try+ perfcheck') name: benchmark runs-on: ubuntu-22.04 continue-on-error: true env: # Cargo binary CARGO_BIN: cargo # When CARGO_BIN is set to CROSS, this is set to `--target matrix.target`. TARGET_FLAGS: "" # When CARGO_BIN is set to CROSS, TARGET_DIR includes matrix.target. TARGET_DIR: ./target # Rustc flags needed by the `all` features RUSTFLAGS: "--cfg tokio_unstable" # SWS features for Cargo build CARGO_FEATURES: "--features=all" # Version of the vegeta binary VEGETA_VERSION: "12.11.1" # Command-line parameters for the vegeta binary VEGETA_FLAGS: "-workers=4 -connections=100 -rate=${{ matrix.rate }} -duration=10s" # Version of the caddy binary CADDY_VERSION: "2.7.6" strategy: fail-fast: false matrix: include: - title: "Small file (dynamic compression)" url: http://localhost:8080/small.txt rate: "1000/s" - title: "Small file (pre-compressed)" url: http://localhost:8080/small_precompressed.txt rate: "1000/s" - title: "Large file (dynamic compression)" url: http://localhost:8080/large.txt rate: "200/s" - title: "Large file (pre-compressed)" url: http://localhost:8080/large_precompressed.txt rate: "500/s" - title: "Directory listing (many files)" url: http://localhost:8080/ rate: "1000/s" - title: "Directory listing (empty directory)" url: http://localhost:8080/emptydir/ rate: "1000/s" - title: "Small directory index file" url: http://localhost:8080/dirsmallindex/ rate: "1000/s" - title: "Large directory index file" url: http://localhost:8080/dirlargeindex/ rate: "200/s" - title: "Not found" url: http://localhost:8080/not_found rate: "1000/s" steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 1 - name: Checkout benchmark-setup repository uses: actions/checkout@v4 with: repository: static-web-server/benchmarks-setup ref: v0.2 path: benchmarks-setup fetch-depth: 1 - name: Cache Rust Cargo/Toolchain uses: actions/cache@v4 with: path: | ~/.cargo **/target key: rust-${{ hashFiles('**/Cargo.lock') }} restore-keys: | rust- - name: Install vegeta binary run: | curl -L https://github.com/tsenart/vegeta/releases/download/v${{ env.VEGETA_VERSION }}/vegeta_${{ env.VEGETA_VERSION }}_linux_amd64.tar.gz | tar xz echo "Downloaded vegeta" ./vegeta --version - name: Install caddy binary run: | curl -L https://github.com/caddyserver/caddy/releases/download/v${{ env.CADDY_VERSION }}/caddy_${{ env.CADDY_VERSION }}_linux_amd64.tar.gz | tar xz caddy echo "Downloaded caddy" ./caddy --version - name: Install web server software uses: awalsh128/cache-apt-pkgs-action@latest with: packages: apache2 lighttpd nginx version: 1.0 - name: Install Rust uses: dtolnay/rust-toolchain@stable with: toolchain: stable target: x86_64-unknown-linux-gnu - name: Show command used for Cargo run: | echo "cargo command is: ${{ env.CARGO_BIN }}" echo "target flag is: ${{ env.TARGET_FLAGS }}" echo "target dir is: ${{ env.TARGET_DIR }}" - name: Build release binary run: ${{ env.CARGO_BIN }} build --bin static-web-server -vv --release ${{ env.CARGO_FEATURES }} ${{ env.TARGET_FLAGS }} - name: Initialize graph.csv run: echo "Sample, min, mean, 50th, 90th, 95th, 99th, max" > graph.csv - name: Run SWS benchmark run: | NAME=$(${{ env.TARGET_DIR }}/release/static-web-server --version | head -n 1 | sed -e 's/^Version: */sws /' -e 's/static-web-server/sws/' -e 's/$/ (src)/') REPORT=report_sws.bin ${{ env.TARGET_DIR }}/release/static-web-server --config-file benchmarks-setup/config/sws.toml & sleep 2 echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT kill %1 ./vegeta report -type=text $REPORT ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv - name: Run Apache benchmark run: | NAME=$(apache2 -v | sed 's/.*: //' | sed 's/ .*//' | sed 's!/! !' | tr '[:upper:]' '[:lower:]') REPORT=report_apache.bin apache2 -D FOREGROUND -f `pwd`/benchmarks-setup/config/apache.conf & sleep 2 echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT kill %1 ./vegeta report -type=text $REPORT ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv - name: Run lighttpd benchmark run: | NAME=$(lighttpd -v | sed 's/ .*//' | sed 's!/! !') REPORT=report_lighttpd.bin lighttpd -D -f `pwd`/benchmarks-setup/config/lighttpd.conf & sleep 2 URL="${{ matrix.url }}" if [[ $URL == *"_precompressed"* ]]; then # lighttpd does not really support pre-compressed files, request them explicitly URL="${{ matrix.url }}.gz" fi echo "GET $URL" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT kill %1 ./vegeta report -type=text $REPORT ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv - name: Run nginx benchmark run: | NAME=$(nginx -v 2>&1 | sed 's/.*: //' | sed 's/ .*//' | sed 's!/! !') REPORT=report_nginx.bin nginx -c `pwd`/benchmarks-setup/config/nginx.conf & sleep 2 echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT kill %1 ./vegeta report -type=text $REPORT ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv - name: Run caddy benchmark run: | NAME="caddy $(./caddy -v | sed 's/ .*//')" REPORT=report_caddy.bin ./caddy run -c `pwd`/benchmarks-setup/config/Caddyfile & sleep 2 echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT kill %1 ./vegeta report -type=text $REPORT ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install pipx run: pip install pipx - name: Produce plot run: | ./vegeta plot --title "${{ matrix.title }}" report_*.bin > detailed_plot.html cat graph.csv cat graph.csv | sed -e 's/,/|/g' -e 's/"//g' -e 's/^/|/' -e 's/$/|/' -e '2 i |-|-|-|-|-|-|-|-|' >> $GITHUB_STEP_SUMMARY pipx run graph-cli --barh --bar-label --xlabel '' --ylabel 'Latency (ms)' --title "${{ matrix.title }}" --figsize 1200x800 --fontsize 10 --ytick-angle 45 --ytick-align top -o overview.png graph.csv - name: Upload benchmark plots uses: actions/upload-artifact@v4 with: name: ${{ matrix.title }} path: | detailed_plot.html overview.png merge-artifacts: if: github.event.issue.pull_request && contains(github.event.comment.body, '/try+ perfcheck') name: merge-artifacts needs: benchmark runs-on: ubuntu-22.04 continue-on-error: true steps: - name: Merge benchmark plots uses: actions/upload-artifact/merge@v4 with: name: Benchmark plots separate-directories: true delete-merged: true