fork/static-web-server

git clone https://git.julia.blue/fork/static-web-server

Jose Quintanarefactor: move perfcheck workflow behind a PR comment trigger (#584)ea9f43f

allow-symlink
8.2 KiB225 linesraw
1name: perfcheck
2on:
3  issue_comment:
4    types: [created, edited]
5    paths:
6      - .github/workflows/perfcheck.yml
7      - .cargo/config.toml
8      - scripts/ci/vegeta_report_to_csv.jq
9      - Cargo.lock
10      - Cargo.toml
11      - src/**
12
13jobs:
14  benchmark:
15    if: github.event.issue.pull_request && contains(github.event.comment.body, '/try+ perfcheck')
16    name: benchmark
17    runs-on: ubuntu-22.04
18    continue-on-error: true
19    env:
20      # Cargo binary
21      CARGO_BIN: cargo
22      # When CARGO_BIN is set to CROSS, this is set to `--target matrix.target`.
23      TARGET_FLAGS: ""
24      # When CARGO_BIN is set to CROSS, TARGET_DIR includes matrix.target.
25      TARGET_DIR: ./target
26      # Rustc flags needed by the `all` features
27      RUSTFLAGS: "--cfg tokio_unstable"
28      # SWS features for Cargo build
29      CARGO_FEATURES: "--features=all"
30      # Version of the vegeta binary
31      VEGETA_VERSION: "12.11.1"
32      # Command-line parameters for the vegeta binary
33      VEGETA_FLAGS: "-workers=4 -connections=100 -rate=${{ matrix.rate }} -duration=10s"
34      # Version of the caddy binary
35      CADDY_VERSION: "2.7.6"
36
37    strategy:
38      fail-fast: false
39      matrix:
40        include:
41        - title: "Small file (dynamic compression)"
42          url: http://localhost:8080/small.txt
43          rate: "1000/s"
44        - title: "Small file (pre-compressed)"
45          url: http://localhost:8080/small_precompressed.txt
46          rate: "1000/s"
47        - title: "Large file (dynamic compression)"
48          url: http://localhost:8080/large.txt
49          rate: "200/s"
50        - title: "Large file (pre-compressed)"
51          url: http://localhost:8080/large_precompressed.txt
52          rate: "500/s"
53        - title: "Directory listing (many files)"
54          url: http://localhost:8080/
55          rate: "1000/s"
56        - title: "Directory listing (empty directory)"
57          url: http://localhost:8080/emptydir/
58          rate: "1000/s"
59        - title: "Small directory index file"
60          url: http://localhost:8080/dirsmallindex/
61          rate: "1000/s"
62        - title: "Large directory index file"
63          url: http://localhost:8080/dirlargeindex/
64          rate: "200/s"
65        - title: "Not found"
66          url: http://localhost:8080/not_found
67          rate: "1000/s"
68
69    steps:
70    - name: Checkout repository
71      uses: actions/checkout@v4
72      with:
73        fetch-depth: 1
74
75    - name: Checkout benchmark-setup repository
76      uses: actions/checkout@v4
77      with:
78        repository: static-web-server/benchmarks-setup
79        ref: v0.2
80        path: benchmarks-setup
81        fetch-depth: 1
82
83    - name: Cache Rust Cargo/Toolchain
84      uses: actions/cache@v4
85      with:
86        path: |
87          ~/.cargo
88          **/target
89        key: rust-${{ hashFiles('**/Cargo.lock') }}
90        restore-keys: |
91          rust-
92
93    - name: Install vegeta binary
94      run: |
95        curl -L https://github.com/tsenart/vegeta/releases/download/v${{ env.VEGETA_VERSION }}/vegeta_${{ env.VEGETA_VERSION }}_linux_amd64.tar.gz | tar xz
96        echo "Downloaded vegeta"
97        ./vegeta --version
98
99    - name: Install caddy binary
100      run: |
101        curl -L https://github.com/caddyserver/caddy/releases/download/v${{ env.CADDY_VERSION }}/caddy_${{ env.CADDY_VERSION }}_linux_amd64.tar.gz | tar xz caddy
102        echo "Downloaded caddy"
103        ./caddy --version
104
105    - name: Install web server software
106      uses: awalsh128/cache-apt-pkgs-action@latest
107      with:
108        packages: apache2 lighttpd nginx
109        version: 1.0
110
111    - name: Install Rust
112      uses: dtolnay/rust-toolchain@stable
113      with:
114        toolchain: stable
115        target: x86_64-unknown-linux-gnu
116
117    - name: Show command used for Cargo
118      run: |
119        echo "cargo command is: ${{ env.CARGO_BIN }}"
120        echo "target flag is: ${{ env.TARGET_FLAGS }}"
121        echo "target dir is: ${{ env.TARGET_DIR }}"
122
123    - name: Build release binary
124      run: ${{ env.CARGO_BIN }} build --bin static-web-server -vv --release ${{ env.CARGO_FEATURES }} ${{ env.TARGET_FLAGS }}
125
126    - name: Initialize graph.csv
127      run: echo "Sample, min, mean, 50th, 90th, 95th, 99th, max" > graph.csv
128
129    - name: Run SWS benchmark
130      run: |
131        NAME=$(${{ env.TARGET_DIR }}/release/static-web-server --version | head -n 1 | sed -e 's/^Version: */sws /' -e 's/static-web-server/sws/' -e 's/$/ (src)/')
132        REPORT=report_sws.bin
133        ${{ env.TARGET_DIR }}/release/static-web-server --config-file benchmarks-setup/config/sws.toml &
134        sleep 2
135        echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
136        kill %1
137        ./vegeta report -type=text $REPORT
138        ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
139
140    - name: Run Apache benchmark
141      run: |
142        NAME=$(apache2 -v | sed 's/.*: //' | sed 's/ .*//' | sed 's!/! !' | tr '[:upper:]' '[:lower:]')
143        REPORT=report_apache.bin
144        apache2 -D FOREGROUND -f `pwd`/benchmarks-setup/config/apache.conf &
145        sleep 2
146        echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
147        kill %1
148        ./vegeta report -type=text $REPORT
149        ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
150
151    - name: Run lighttpd benchmark
152      run: |
153        NAME=$(lighttpd -v | sed 's/ .*//' | sed 's!/! !')
154        REPORT=report_lighttpd.bin
155        lighttpd -D -f `pwd`/benchmarks-setup/config/lighttpd.conf &
156        sleep 2
157        URL="${{ matrix.url }}"
158        if [[ $URL == *"_precompressed"* ]]; then
159          # lighttpd does not really support pre-compressed files, request them explicitly
160          URL="${{ matrix.url }}.gz"
161        fi
162        echo "GET $URL" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
163        kill %1
164        ./vegeta report -type=text $REPORT
165        ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
166
167    - name: Run nginx benchmark
168      run: |
169        NAME=$(nginx -v 2>&1 | sed 's/.*: //' | sed 's/ .*//' | sed 's!/! !')
170        REPORT=report_nginx.bin
171        nginx -c `pwd`/benchmarks-setup/config/nginx.conf &
172        sleep 2
173        echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
174        kill %1
175        ./vegeta report -type=text $REPORT
176        ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
177
178    - name: Run caddy benchmark
179      run: |
180        NAME="caddy $(./caddy -v | sed 's/ .*//')"
181        REPORT=report_caddy.bin
182        ./caddy run -c `pwd`/benchmarks-setup/config/Caddyfile &
183        sleep 2
184        echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
185        kill %1
186        ./vegeta report -type=text $REPORT
187        ./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
188
189    - name: Set up Python
190      uses: actions/setup-python@v5
191      with:
192        python-version: '3.12'
193
194    - name: Install pipx
195      run: pip install pipx
196
197    - name: Produce plot
198      run: |
199        ./vegeta plot --title "${{ matrix.title }}" report_*.bin > detailed_plot.html
200        cat graph.csv
201        cat graph.csv | sed -e 's/,/|/g' -e 's/"//g' -e 's/^/|/' -e 's/$/|/' -e '2 i |-|-|-|-|-|-|-|-|' >> $GITHUB_STEP_SUMMARY
202        pipx run graph-cli --barh --bar-label --xlabel '' --ylabel 'Latency (ms)' --title "${{ matrix.title }}" --figsize 1200x800 --fontsize 10 --ytick-angle 45 --ytick-align top -o overview.png graph.csv
203
204    - name: Upload benchmark plots
205      uses: actions/upload-artifact@v4
206      with:
207        name: ${{ matrix.title }}
208        path: |
209          detailed_plot.html
210          overview.png
211
212  merge-artifacts:
213    if: github.event.issue.pull_request && contains(github.event.comment.body, '/try+ perfcheck')
214    name: merge-artifacts
215    needs: benchmark
216    runs-on: ubuntu-22.04
217    continue-on-error: true
218
219    steps:
220    - name: Merge benchmark plots
221      uses: actions/upload-artifact/merge@v4
222      with:
223        name: Benchmark plots
224        separate-directories: true
225        delete-merged: true