diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..084871cb --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,152 @@ +name: Continuous Integration + +on: + push: + branches: [ "master" ] + paths: + - "libfaac/**" + - ".github/workflows/benchmark.yml" + pull_request: + branches: [ "*" ] + paths: + - "libfaac/**" + - ".github/workflows/benchmark.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + name: Benchmark ${{ matrix.arch }} / ${{ matrix.precision }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + arch: [amd64] + precision: [single, double] + + steps: + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y meson ninja-build bc ffmpeg + + - name: Checkout Candidate + uses: actions/checkout@v4 + with: + path: candidate + + - name: Build Candidate + run: | + cd candidate + meson setup build_cand -Dfloating-point=${{ matrix.precision }} --buildtype=release + ninja -C build_cand + + - name: Determine Baseline SHA + id: baseline-sha + run: | + if [ "${{ github.event_name }}" == "push" ]; then + echo "sha=${{ github.sha }}" >> $GITHUB_OUTPUT + else + echo "sha=${{ github.event.pull_request.base.sha }}" >> $GITHUB_OUTPUT + fi + + - name: Restore Baseline Results + id: cache-baseline + uses: actions/cache/restore@v4 + with: + path: results/${{ matrix.arch }}_${{ matrix.precision }}_base.json + key: ${{ runner.os }}-baseline-${{ matrix.arch }}-${{ matrix.precision }}-${{ steps.baseline-sha.outputs.sha }} + + - name: Checkout Baseline + if: steps.cache-baseline.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + ref: ${{ steps.baseline-sha.outputs.sha }} + path: baseline + + - name: Build Baseline + if: steps.cache-baseline.outputs.cache-hit != 'true' + run: | + cd baseline + meson setup build_base -Dfloating-point=${{ matrix.precision }} --buildtype=release + ninja -C build_base + + - name: Run Benchmark (Baseline) + if: steps.cache-baseline.outputs.cache-hit != 'true' + uses: nschimme/faac-benchmark@v1 + with: + faac-bin: ./baseline/build_base/frontend/faac + libfaac-so: ./baseline/build_base/libfaac/libfaac.so + run-name: ${{ matrix.arch }}_${{ matrix.precision }}_base + output-json: ./results/${{ matrix.arch }}_${{ matrix.precision }}_base.json + visqol-image: ghcr.io/nschimme/faac-benchmark-visqol:latest + + - name: Save Baseline Results + if: success() && steps.cache-baseline.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: results/${{ matrix.arch }}_${{ matrix.precision }}_base.json + key: ${{ runner.os }}-baseline-${{ matrix.arch }}-${{ matrix.precision }}-${{ steps.baseline-sha.outputs.sha }} + + - name: Run Benchmark (Candidate) + if: github.event_name == 'pull_request' + uses: nschimme/faac-benchmark@v1 + with: + faac-bin: ./candidate/build_cand/frontend/faac + libfaac-so: ./candidate/build_cand/libfaac/libfaac.so + run-name: ${{ matrix.arch }}_${{ matrix.precision }}_cand + output-json: ./results/${{ matrix.arch }}_${{ matrix.precision }}_cand.json + visqol-image: ghcr.io/nschimme/faac-benchmark-visqol:latest + + - name: Upload Results + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.arch }}-${{ matrix.precision }} + path: results/*.json + + report: + name: Consolidated Report + needs: benchmark + runs-on: ubuntu-latest + if: always() && github.event_name == 'pull_request' + permissions: + pull-requests: write + steps: + - name: Checkout Code + if: github.event_name == 'pull_request' + uses: actions/checkout@v4 + + - name: Download all results + uses: actions/download-artifact@v4 + with: + path: results + pattern: results-* + merge-multiple: true + + - name: Generate Report + id: generate + uses: nschimme/faac-benchmark/report@v1 + with: + results-path: ./results + base-sha: ${{ github.event.pull_request.base.sha }} + cand-sha: ${{ github.event.pull_request.head.sha }} + summary-only: false + + - name: Upload Full Report + uses: actions/upload-artifact@v4 + with: + name: benchmark-report-full + path: report.md + + - name: Post Summary to PR + if: always() && github.event_name == 'pull_request' + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr comment ${{ github.event.pull_request.number }} --body-file summary.md + diff --git a/README b/README index 2c30f925..ccce8022 100644 --- a/README +++ b/README @@ -79,3 +79,12 @@ General FAAC compiling instructions cd build meson setup .. meson install + +___________________________________ +Benchmarking + +FAAC uses a dedicated benchmark suite to ensure quality and performance. +The suite is hosted in a separate repository: https://github.com/nschimme/faac-benchmark + +Automated benchmarks run on every pull request. For instructions on how to +run benchmarks locally, please refer to the README in the benchmark repository.