diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f65506f..2ed5264 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,20 +1,20 @@ -# packcheck-0.7.0 +# packcheck-0.7.1 # You can use any of the options supported by packcheck as environment # variables here. See https://github.com/composewell/packcheck for all # options and their explanation. -name: packcheck +name: TEST #----------------------------------------------------------------------------- # Events on which the build should be triggered #----------------------------------------------------------------------------- on: + workflow_dispatch: + pull_request: push: branches: - master - pull_request: - #----------------------------------------------------------------------------- # Build matrix @@ -22,158 +22,492 @@ on: jobs: build: - name: ${{ matrix.name }} + name: >- + ${{ matrix.runner }} + ${{ matrix.command }} + ${{ matrix.ghc_version }} + ${{ matrix.name }} env: + # ------------------------------------------------------------------------ + # What to build + # ------------------------------------------------------------------------ + # DISABLE_TEST: "y" + # DISABLE_BENCH: "y" + # DISABLE_DOCS: "y" + # DISABLE_SDIST_BUILD: "y" + # DISABLE_SDIST_GIT_CHECK: "y" + # DISABLE_DIST_CHECKS: "y" + + # ------------------------------------------------------------------------ + # Selecting tool versions + # ------------------------------------------------------------------------ + # For updating see: https://downloads.haskell.org/~ghcup/ + #GHCUP_VERSION: 0.1.50.2 + + # ------------------------------------------------------------------------ + # cabal options + # ------------------------------------------------------------------------ + + # CABAL_BUILD_OPTIONS="-j1" + # Enable the above option if you: + # 1. want logs in serial order to be able to correlate them better. + # + # 2. run into memory issues due to Build paralellism. We need to + # have sufficient per CPU memory, if not we can use a lower -j + # option; cabal default is ncpus. + # + # Currently (2026-03-12) from packcheck output we see: + # Linux: 15GB, 4 cpus + # macOS: 7GB, 3 cpus + # Windows: 16GB, 4 cpus + + # ------------------------------------------------------------------------ + # stack options (if using stack builds) + # ------------------------------------------------------------------------ + # Note requiring a specific version of stack using STACKVER may fail due to + # github API limit while checking and upgrading/downgrading to the specific + # version. + #STACKVER: "1.6.5" + #STACK_UPGRADE: "y" + # ------------------------------------------------------------------------ # Common options # ------------------------------------------------------------------------ - # GHC_OPTIONS: "-Werror" - GHCUP_VERSION: 0.1.20.0 - DOCSPEC_URL: https://github.com/phadej/cabal-extras/releases/download/cabal-docspec-0.0.0.20210111/cabal-docspec-0.0.0.20210111.xz - DOCSPEC_OPTIONS: "--timeout 60" - CABAL_REINIT_CONFIG: y - LC_ALL: C.UTF-8 - STACK_UPGRADE: "y" + # Subdir + SUBDIR: ${{ matrix.subdir }} - CABAL_CHECK_RELAX: y - CABAL_HACKAGE_MIRROR: hackage.haskell.org:http://hackage.fpcomplete.com + # If this option is enabled packcheck clears out the environment + # and only uses the options explicitly set on the command line, + # implicit environment won't affect the build. if you find this + # annoying comment this. + CHECK_ENV: "y" - PACKCHECK: "./packcheck.sh" + # NOTE: by capturing the options in env variables here quotes + # are presrved, if we expand them directly into the script + # quotes are lost. + # + # WARNING! you cannot use comments in this + COMMON_OPTIONS: >- + LC_ALL=C.UTF-8 + BUILD_PREFETCH=y + GHCUP_VERSION=latest + GHCVER=${{ matrix.ghc_version }} + + # WARNING! you cannot use comments in this + CABAL_OPTIONS: >- + CABAL_REINIT_CONFIG=y + CABAL_CHECK_RELAX=y + + MATRIX_OPTIONS: ${{ matrix.pack_options }} + + # ------------------------------------------------------------------------ + # Location of packcheck.sh (the shell script invoked to perform CI tests ). + # ------------------------------------------------------------------------ + # You can either commit the packcheck.sh script in your repo root or + # you can use it by specifying the PACKCHECK_REPO_URL option. + # If you have not committed packcheck.sh in your repo at PACKCHECK + # then it is automatically pulled from this URL. PACKCHECK_GITHUB_URL: "https://raw.githubusercontent.com/composewell/packcheck" - PACKCHECK_GITHUB_COMMIT: "7c4e1ab7a59b5ed6e92cfd0da67460a6116be4ac" - - BUILD: ${{ matrix.build }} - GHCVER: ${{ matrix.ghc_version }} - #RESOLVER: ${{ matrix.resolver }} - CABAL_BUILD_OPTIONS: >- - -f drinkery - -f conduit - -f pipes - -f machines - -f streaming - ${{ matrix.cabal_build_options }} - CABAL_PROJECT: ${{ matrix.cabal_project }} - DISABLE_DOCS: ${{ matrix.disable_docs }} - #DISABLE_SDIST_BUILD: ${{ matrix.disable_sdist_build }} - #DISABLE_SDIST_BUILD: "y" - HLINT_VERSION: 3.6.1 - HLINT_OPTIONS: ${{ matrix.hlint_options }} - HLINT_TARGETS: ${{ matrix.hlint_targets }} + PACKCHECK_GITHUB_COMMIT: "b91dd7daf50d91047ca8c1cce47e8634e34dfcbc" + # ubuntu seems to have better support than debian on CI systems runs-on: ${{ matrix.runner }} + #continue-on-error: ${{ matrix.ignore_error }} strategy: fail-fast: false matrix: + + # The order of jobs is important to optimize fail-fast, if + # fail-fast is set to true. + + # This section is to order the important jobs first especially for + # "fail-fast" so that these are the ones started first. + #name: + # - ci + + # IMPORTANT NOTE: + # If runner, command, ghc version are identical then the name of + # CI is used to distinguish the cache key. + + # The reason we have an explicit "name" field in the following + # entries is to force an additional config instead of adding to + # an existing config while adding additional configs. + # Look at + # for more info about adding matrix elements. + # Adding any element to the list will increase the number of matrix + # elements proportional to the cross product. + include: - - name: 9.10.1-Werror - ghc_version: 9.10.1 - build: cabal + + #- name: ci + # runner: ubuntu-latest + # command: cabal + # ghc_version: head + # # The URL may change, to find a working URL go to https://gitlab.haskell.org/ghc/ghc/-/jobs/ + # # Find a debian10/11/12 job, click on a passed/failed status, at the + # # end of the output you will find the tar.xz name, put that tar + # # name after "raw/", and put the job name after "job=". + # # Also see https://github.com/mpickering/ghc-artefact-nix/blob/master/gitlab-artifact.nix + # # + # # May also use ghcup for installing ghc head version, use the + # # version "LatestNightly", and the following config: + # # ghcup config add-release-channel https://ghc.gitlab.haskell.org/ghcup-metadata/ghcup-nightlies-0.0.7.yaml + # # WARNING! cannot use # comments inside pack_options. + # pack_options: >- + # GHCUP_GHC_OPTIONS="-u https://gitlab.haskell.org/ghc/ghc/-/jobs/artifacts/master/raw/ghc-x86_64-linux-deb10-int_native-validate.tar.xz?job=x86_64-linux-deb10-int_native-validate" + # CABAL_PROJECT=cabal.project.ghc-head + # DISABLE_SDIST_BUILD="y" + + - name: ci runner: ubuntu-latest - cabal-version: 3.10.2.0 - cabal_project: cabal.project.Werror + command: cabal + ghc_version: 9.14.1 + # WARNING! cannot use # comments inside pack_options. + # CABAL_BUILD_OPTIONS="-f drinkery -f conduit -f pipes -f machines -f streaming" + pack_options: >- + CABAL_PROJECT=cabal.project.user - - name: 9.8.1-Werror - ghc_version: 9.8.1 - build: cabal + - name: bench-runner runner: ubuntu-latest - cabal-version: 3.10.2.0 - cabal_project: cabal.project.Werror + command: cabal + ghc_version: 9.12.4 + subdir: bench-runner + pack_options: >- + CABAL_PROJECT=cabal.project.user - - name: hlint - build: hlint - hlint_options: "lint" - hlint_targets: "lib Benchmarks" + - name: ci runner: ubuntu-latest - cabal-version: 3.2 - cabal_project: cabal.project.user + command: cabal + ghc_version: 9.12.4 + # Removed "-f pipes" and machines as the build was failing with heap overflow + pack_options: >- + CABAL_PROJECT=cabal.project.user + CABAL_BUILD_OPTIONS="-f drinkery -f conduit -f streaming" - - name: 9.6.3-macos - runner: macos-latest - ghc_version: 9.6.3 - build: cabal - cabal-version: 3.10.1.0 - cabal_project: cabal.project.user + - name: Werror + runner: ubuntu-latest + command: cabal + ghc_version: 9.10.3 + # WARNING! cannot use # comments inside pack_options. + pack_options: >- + CABAL_PROJECT=cabal.project.Werror + CABAL_BUILD_OPTIONS="-f drinkery -f conduit -f pipes -f machines -f streaming" - - name: 9.4.4-no-fusion-plugin + - name: ci runner: ubuntu-latest - ghc_version: 9.4.4 - build: cabal - cabal-version: 3.8.1.0 - cabal_project: cabal.project.user - cabal_build_options: -f no-fusion-plugin + command: cabal + ghc_version: 9.8.4 + # WARNING! cannot use # comments inside pack_options. + pack_options: >- + CABAL_PROJECT=cabal.project.user + CABAL_BUILD_OPTIONS="-f drinkery -f conduit -f pipes -f machines -f streaming" - - name: 9.4.4 + - name: ci + runner: macos-latest + command: cabal + ghc_version: 9.6.3 + # WARNING! cannot use # comments inside pack_options. + pack_options: >- + CABAL_PROJECT=cabal.project.user + CABAL_BUILD_OPTIONS="-f drinkery -f conduit -f pipes -f machines -f streaming" + + - name: no-fusion-plugin runner: ubuntu-latest + command: cabal ghc_version: 9.4.4 - build: cabal - cabal-version: 3.8.1.0 - cabal_project: cabal.project.user + pack_options: >- + CABAL_PROJECT=cabal.project.user + CABAL_BUILD_OPTIONS="-f drinkery -f conduit -f pipes -f machines -f streaming -f no-fusion-plugin" - - name: 9.2.7 + - name: ci runner: ubuntu-latest - ghc_version: 9.2.7 - build: cabal - cabal-version: 3.6 - cabal_project: cabal.project.user + command: cabal + ghc_version: 9.2.8 + pack_options: >- + CABAL_PROJECT=cabal.project.user + CABAL_BUILD_OPTIONS="-f drinkery -f conduit -f pipes -f machines -f streaming" - - name: 8.10.7 + - name: ci runner: ubuntu-latest + command: cabal ghc_version: 8.10.7 - build: cabal - cabal-version: 3.2 - cabal_project: cabal.project.user - disable_docs: y + # WARNING! cannot use # comments inside pack_options. + pack_options: >- + CABAL_PROJECT=cabal.project.user + DISABLE_DOCS=y + CABAL_BUILD_OPTIONS="-f drinkery -f conduit -f pipes -f machines -f streaming" - - name: 8.8.4 + - name: ci runner: ubuntu-latest - ghc_version: 8.8.4 - build: cabal - cabal-version: 3.2 - cabal_project: cabal.project.user - disable_docs: y + command: hlint + # WARNING! cannot use # comments inside pack_options. + pack_options: >- + HLINT_VERSION=3.6.1 + HLINT_OPTIONS="lint" + HLINT_TARGETS="lib Benchmarks" - - name: 8.6.5 - runner: ubuntu-latest - ghc_version: 8.6.5 - build: cabal - cabal-version: 3.2 - cabal_project: cabal.project.user - disable_docs: y +#----------------------------------------------------------------------------- +# Usually you do not need to change anything below, this is generic code for +# caching and running packcheck on Linux/Mac/Windows. +#----------------------------------------------------------------------------- steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v4 - name: Restore cache + - name: Optimize Windows Runner Environment + if: runner.os == 'Windows' + shell: powershell + run: | + # Enable long paths. Sometimes failures occur due to path + # length limitation. + Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 1 + + # Cache saving is very slow on Windows, it could be due to tar, + # or the compression program or the file system. Changing the + # compression level of zstd does not help. We are also not able to pick + # another tar program by changing the path, tar is picked up by the + # cache action itself. + + # This does not seem to make much difference. + # Disable Windows Defender to speed up IO/Tar operations + Set-MpPreference -DisableRealtimeMonitoring $true + + - uses: actions/checkout@v4 + + # This should happen before cache restore. + - name: Remove ~/.ghcup symlink (non-Windows) + if: runner.os != 'Windows' + run: | + rm -f ~/.ghcup + + # See the "cabal path" output in the CI logs to tweak the cache + # locations. + + # Shared by all tests, all platforms + - name: Restore hackage package index (non-Windows) + id: restore-hackage + uses: actions/cache/restore@v4 + if: runner.os != 'Windows' with: path: | - ~/.local - ~/.cabal + ~/.cache/cabal/packages + key: cabal-hackage-index-non-windows + + # GHC is not test/options specific + - name: Restore ghcup and ghc (non-Windows) + id: restore-ghcup + uses: actions/cache/restore@v4 + if: runner.os != 'Windows' + with: + path: | + ~/.ghcup + key: ghc-${{ matrix.ghc_version }}-v${{ matrix.ghc_salt }}-${{ matrix.runner }} + + # GHC is not test/options specific + #- name: Restore ghcup and ghc (Windows) + # id: restore-ghcup-win + # uses: actions/cache/restore@v4 + # if: runner.os == 'Windows' + # with: + # path: | + # C:/ghcup + # key: ghc-${{ matrix.ghc_version }}-v${{ matrix.ghc_salt }}-${{ matrix.runner }} + + # XXX If we want to invalidate the cache on resolver change we + # should use a separate cache for stack as it should be keyed on + # resolver as well. But that requires resolver as a matrix element + # and then copied to pack_options from there, needs some work. It + # should be fine as resolver minor version change can only increase + # the cache size. + # Deps are test/options specific + - name: Restore build dependencies (non-Windows) + id: restore-deps-nonwin + uses: actions/cache/restore@v4 + if: runner.os != 'Windows' + with: + # cabal: ~/.local/bin, ~/.local/state/cabal + # ~/.cache/cabal/packages can be cached, but will increase the + # cache size. But can save the "cabal update" time. + path: | + ~/.local/bin + ~/.local/state/cabal ~/.stack - .stack-work - key: ${{ runner.os }}-${{ matrix.name }} + key: ${{ matrix.command }}-deps-ghc-${{ matrix.ghc_version }}-v${{ matrix.ghc_salt }}-${{ matrix.runner }}-${{ matrix.name }} + + # Deps are test/options specific + - name: Restore build dependencies, hackage index, local/bin (Windows) + id: restore-deps-win + uses: actions/cache/restore@v4 + if: runner.os == 'Windows' + with: + path: | + C:/cabal + key: ${{ matrix.command }}-deps-ghc-${{ matrix.ghc_version }}-v${{ matrix.ghc_salt }}-${{ matrix.runner }}-${{ matrix.name }} - name: Download packcheck + # on windows-latest GitHub Actions defaults to PowerShell + shell: bash run: | - # If a custom stack-yaml is specified, replace the default with that - #if test -e "$STACK_YAML"; then rm -f stack.yaml && ln -sv $STACK_YAML stack.yaml; else true; fi - #unset STACK_YAML - - if test ! -e "$PACKCHECK" - then - if test -z "$PACKCHECK_GITHUB_COMMIT" + download_one () { + if test ! -e "$1" then - die "PACKCHECK_GITHUB_COMMIT is not specified." + if test -z "$PACKCHECK_GITHUB_COMMIT" + then + die "Downloading [$1]: PACKCHECK_GITHUB_COMMIT env var is not specified." + fi + PACKCHECK_URL_PREFIX=${PACKCHECK_GITHUB_URL}/${PACKCHECK_GITHUB_COMMIT} + curl --fail -sLO ${PACKCHECK_URL_PREFIX}/$1 || exit 1 + chmod +x "$1" + elif test ! -x "$1" + then + chmod +x $1 fi - PACKCHECK_URL=${PACKCHECK_GITHUB_URL}/${PACKCHECK_GITHUB_COMMIT}/packcheck.sh - curl --fail -sL -o "$PACKCHECK" $PACKCHECK_URL || exit 1 - chmod +x $PACKCHECK - elif test ! -x "$PACKCHECK" - then - chmod +x $PACKCHECK + } + if test -n "$SUBDIR"; then + cd "$SUBDIR" fi + download_one packcheck.sh - - name: Run packcheck + # Store the function in a file so that it can be reused across multiple + # steps. + - name: Setup packcheck + shell: bash run: | - bash -c "$PACKCHECK $BUILD" + if test -n "$SUBDIR"; then + rm -f cabal.project + cd "$SUBDIR" + fi + cat << 'EOF' > packcheck-setup.sh + setup_packcheck_args() { + ARGS="${{ matrix.command }} $COMMON_OPTIONS" + if test "${{ matrix.command }}" = "cabal"; then + ARGS="$ARGS $CABAL_OPTIONS" + fi + + # Caution: do not use matrix.pack_options directly here + # quotes will get stripped. + ARGS="$ARGS $MATRIX_OPTIONS" + PATH_VAR=/bin:/usr/bin:/sbin:/usr/sbin + + case "$(uname)" in + Linux) + export GHCUP_INSTALL_BASE_PREFIX=$HOME + # Access usr/local seems to be much slower on github + #PATH_VAR="$PATH_VAR:/usr/local/.ghcup/bin" + ;; + CYGWIN*|MINGW*|MSYS*) + for var in APPDATA CABAL_DIR; do + val="${!var}" + [ -n "$val" ] && ARGS="$ARGS $var=\"$(cygpath -u "$val")\"" + done + ARGS="$ARGS CABAL_BUILDDIR=\"/c/b\" WINVER=$WINVER" + ARGS="$ARGS NUMBER_OF_PROCESSORS=$NUMBER_OF_PROCESSORS" + ARGS="$ARGS PROCESSOR_IDENTIFIER=\"$PROCESSOR_IDENTIFIER\"" + PATH_VAR="$PATH_VAR:/c/ghcup:/c/Program Files/7-Zip:/mingw64/bin" + ;; + esac + ARGS="$ARGS PATH=\"$PATH_VAR\"" + export ARGS + } + EOF + + - name: Run packcheck (dependencies only) + id: deps-only + # Check the relevant step based on the runner OS + if: | + (runner.os != 'Windows' && steps.restore-deps-nonwin.outputs.cache-hit != 'true') || + (runner.os == 'Windows' && steps.restore-deps-win.outputs.cache-hit != 'true') + shell: bash + run: | + if test -n "$SUBDIR"; then + cd "$SUBDIR" + fi + source packcheck-setup.sh + setup_packcheck_args + eval "./packcheck.sh $ARGS BUILD_DEPS_ONLY=y" + + - name: Save hackage package index (non-Windows) + uses: actions/cache/save@v4 + if: always() && steps.deps-only.outcome == 'success' && runner.os != 'Windows' && steps.restore-hackage.outputs.cache-hit != 'true' + with: + path: ~/.cache/cabal/packages + key: cabal-hackage-index-non-windows + + - name: Save ghcup and ghc (non-Windows) + uses: actions/cache/save@v4 + if: always() && steps.deps-only.outcome == 'success' && runner.os != 'Windows' && steps.restore-ghcup.outputs.cache-hit != 'true' + with: + path: ~/.ghcup + key: ghc-${{ matrix.ghc_version }}-v${{ matrix.ghc_salt }}-${{ matrix.runner }} + + # On Windows, ghc restore step takes 2-3 minutes whereas installing + # GHC takes less than 2 minutes. On top of this saving the ghc cache + # takes 7 minutes. So this does not make sense. + #- name: Save ghcup and ghc (Windows) + # uses: actions/cache/save@v4 + # if: always() && steps.deps-only.outcome == 'success' && runner.os == 'Windows' && steps.restore-ghcup-win.outputs.cache-hit != 'true' + # with: + # path: | + # C:/ghcup + # key: ghc-${{ matrix.ghc_version }}-v${{ matrix.ghc_salt }}-${{ matrix.runner }} + + # Deps are test/options specific, hence, matrix.name in the key + - name: Save build dependencies (non-Windows) + uses: actions/cache/save@v4 + if: always() && steps.deps-only.outcome == 'success' && runner.os != 'Windows' && steps.restore-deps-nonwin.outputs.cache-hit != 'true' + with: + path: | + ~/.local/bin + ~/.local/state/cabal + ~/.stack + key: ${{ matrix.command }}-deps-ghc-${{ matrix.ghc_version }}-v${{ matrix.ghc_salt }}-${{ matrix.runner }}-${{ matrix.name }} + + # hackage package index is part of C:/cabal + # Deps are test/options specific, hence, matrix.name in the key + # Add if needed ~/AppData/Roaming/local + - name: Save build dependencies (Windows) + uses: actions/cache/save@v4 + if: always() && steps.deps-only.outcome == 'success' && runner.os == 'Windows' && steps.restore-deps-win.outputs.cache-hit != 'true' + with: + path: | + C:/cabal + key: ${{ matrix.command }}-deps-ghc-${{ matrix.ghc_version }}-v${{ matrix.ghc_salt }}-${{ matrix.runner }}-${{ matrix.name }} + + - name: Run packcheck (final build) + shell: bash + run: | + if test -n "$SUBDIR"; then + cd "$SUBDIR" + fi + source packcheck-setup.sh + setup_packcheck_args + eval "./packcheck.sh $ARGS BUILD_PACKAGE_ONLY=y" + + # Enable for debugging, this itself takes 50 sec on Windows, in packcheck + # build which is tiny. + #- name: Check cache locations + # shell: bash + # run: | + # # We want to see if any cache paths are symlinks; e.g. on github + # # ~/.ghcup is a pre-existing symlink to /usr/local. + # if [ "$RUNNER_OS" == "Windows" ]; then + # # Convert Windows paths to Unix-style for Bash + # APPDATA_UNIX=$(cygpath "$APPDATA") + # list="$APPDATA_UNIX/local C:/ghcup C:/cabal" + # else + # list="$HOME/.local/bin $HOME/.local/state/cabal $HOME/.ghcup $HOME/.stack" + # fi + + # for dir in $list; do + # # macOS does not have --no-dereference option + # if [ -L "$dir" ]; then + # echo "$dir is a symlink -> $(readlink $dir)" + # elif [ -d "$dir" ]; then + # du -sh "$dir" 2>/dev/null || echo "$dir permission denied" + # else + # echo "$dir missing or not a dir" + # fi + # done + # echo + # #ls -ld $list 2>/dev/null || true + # ls -al $list 2>/dev/null || true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..48a004c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +dist-newstyle diff --git a/.packcheck.ignore b/.packcheck.ignore index 71a32fe..6b6d362 100644 --- a/.packcheck.ignore +++ b/.packcheck.ignore @@ -1,4 +1,5 @@ .github/workflows/build.yml +.gitignore .packcheck.ignore Benchmarks/ListT.hs Benchmarks/ListTransformer.hs @@ -7,5 +8,7 @@ Benchmarks/SimpleConduit.hs appveyor.yml cabal.project.user cabal.project.Werror -default.nix -stack.yaml +flake.lock +flake.nix +packages.nix +sources.nix diff --git a/Changelog.md b/Changelog.md index f312042..b599d1d 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,7 @@ +## 0.4.1 + +* Support streamly-core-0.3.0 and streamly-0.11.0 + ## 0.4.0 * Add different streaming libraries under individual flags diff --git a/README.md b/README.md index c685246..9e9b309 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This package provides micro-benchmarks to measure and compare the performance of various streaming implementations in Haskell. -We have taken due to care to make sure that we are +We have taken due care to make sure that we are benchmarking correctly and fairly. See [the notes on correct benchmarking](docs/benchmarking-notes.md). diff --git a/appveyor.yml b/appveyor.yml index f54f8b6..7cbbc60 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,8 +1,12 @@ -# packcheck-0.4.2 +# packcheck-0.7.1 # You can use any of the options supported by packcheck as environment # variables here. See https://github.com/composewell/packcheck for all # options and their explanation. +#branches: +# only: +# - master + environment: # ------------------------------------------------------------------------ # Global options, you can use these per build as well @@ -11,42 +15,34 @@ environment: # ------------------------------------------------------------------------ # Common options # ------------------------------------------------------------------------ - # GHC_OPTIONS: "-Werror" CABAL_REINIT_CONFIG: "y" LC_ALL: "C.UTF-8" + # ------------------------------------------------------------------------ + # How to build + # ------------------------------------------------------------------------ + # + GHCUP_VERSION: "0.1.50.2" + GHCVER: "9.10.3" + #CABALVER: "3.10.3.0" + # ------------------------------------------------------------------------ # What to build # ------------------------------------------------------------------------ # DISABLE_TEST: "y" # DISABLE_BENCH: "y" # DISABLE_DOCS: "y" - DISABLE_SDIST_BUILD: "y" # DISABLE_DIST_CHECKS: "y" - - # ------------------------------------------------------------------------ - # stack options - # ------------------------------------------------------------------------ - # Note requiring a specific version of stack using STACKVER may fail due to - # github API limit while checking and upgrading/downgrading to the specific - # version. - #STACKVER: "1.6.5" - STACK_UPGRADE: "y" - RESOLVER: "lts-21.25" - STACK_ROOT: "c:\\sr" + # DISABLE_SDIST_BUILD: "y" + # Note: these require the "diff" utility. + # DISABLE_SDIST_GIT_CHECK: "y" + DISABLE_SDIST_PROJECT_CHECK: "y" # ------------------------------------------------------------------------ # cabal options # ------------------------------------------------------------------------ CABAL_CHECK_RELAX: "y" - CABAL_NO_SANDBOX: "y" - CABAL_HACKAGE_MIRROR: "hackage.haskell.org:http://hackage.fpcomplete.com" - - # ------------------------------------------------------------------------ - # Where to find the required tools - # ------------------------------------------------------------------------ - PATH: "%PATH%;%APPDATA%\\local\\bin" - LOCAL_BIN: "%APPDATA%\\local\\bin" + #CABAL_PROJECT: "cabal.project" # ------------------------------------------------------------------------ # Location of packcheck.sh (the shell script invoked to perform CI tests ). @@ -59,31 +55,33 @@ environment: # If you have not committed packcheck.sh in your repo at PACKCHECK_LOCAL_PATH # then it is automatically pulled from this URL. PACKCHECK_GITHUB_URL: "https://raw.githubusercontent.com/composewell/packcheck" - PACKCHECK_GITHUB_COMMIT: "79fb4437009a7ebdada33d0493c27ee30160ec3f" + PACKCHECK_GITHUB_COMMIT: "3d3c94bf48f18a99271d509c1f1702d15824b1f0" # Override the temp directory to avoid sed escaping issues # See https://github.com/haskell/cabal/issues/5386 TMP: "c:\\tmp" +# Bump the -> version to clear the cache +# packcheck uses "%APPDATA%\\local" to install tools like hlint etc. +# cabal may use "%APPDATA%\\cabal" or "c:\\cabal" +# ghcup may use "%APPDATA%\\ghcup" or "c:\\ghcup" cache: - - "%STACK_ROOT%" - - "%LOCAL_BIN%" + - "%APPDATA%\\local\\bin -> v1" - "%APPDATA%\\cabal" - - "%APPDATA%\\ghc" -# - "%LOCALAPPDATA%\\Programs\\stack" + - "%LOCALAPPDATA%\\cabal" + - "C:\\ghcup" + - "C:\\cabal" +# Folder where the repository is cloned. clone_folder: "c:\\pkg" build: off before_test: -- if not exist %PACKCHECK_LOCAL_PATH% curl -sSkL -o%PACKCHECK_LOCAL_PATH% %PACKCHECK_GITHUB_URL%/%PACKCHECK_GITHUB_COMMIT%/packcheck.sh -- if not exist %LOCAL_BIN% mkdir %LOCAL_BIN% -- where stack.exe || curl -sSkL -ostack.zip http://www.stackage.org/stack/windows-x86_64 && 7z x stack.zip stack.exe && move stack.exe %LOCAL_BIN% -- if defined STACKVER (stack upgrade --binary-only --binary-version %STACKVER%) else (stack upgrade --binary-only || ver > nul) -- stack --version +- if not exist %PACKCHECK_LOCAL_PATH% curl --fail -sSL -o%PACKCHECK_LOCAL_PATH% %PACKCHECK_GITHUB_URL%/%PACKCHECK_GITHUB_COMMIT%/packcheck.sh test_script: -- stack setup > nul - for /f "usebackq tokens=*" %%i in (`where 7z.exe`) do set PATH7Z=%%i\.. - for /f "usebackq tokens=*" %%i in (`where git.exe`) do set PATHGIT=%%i\.. -- chcp 65001 && stack exec bash -- -c "chmod +x %PACKCHECK_LOCAL_PATH%; %PACKCHECK_LOCAL_PATH% stack PATH=/usr/bin:\"%PATH7Z%\":\"%PATHGIT%\"" +- for /f "usebackq tokens=*" %%i in (`where curl.exe`) do set PATHCURL=%%i\.. +- chcp 65001 +- bash %PACKCHECK_LOCAL_PATH% cabal PATH="/usr/bin:%PATH7Z%:%PATHGIT%:%PATHCURL%" diff --git a/bench-runner/cabal.project.user b/bench-runner/cabal.project.user index 23c6b56..c435838 100644 --- a/bench-runner/cabal.project.user +++ b/bench-runner/cabal.project.user @@ -1,35 +1,35 @@ packages: . -source-repository-package - type: git - location: https://github.com/composewell/streamly.git - tag: 9a039b89c6b81300b2bd9c8765bf4bfa18e7c83c - -source-repository-package - type: git - location: https://github.com/composewell/streamly.git - tag: 9a039b89c6b81300b2bd9c8765bf4bfa18e7c83c - subdir: core - -source-repository-package - type: git - location: https://github.com/composewell/streamly-process.git - tag: c1ce40ebe84973d0c54dd27fb17e337bf9ca4a46 - -source-repository-package - type: git - location: https://github.com/composewell/streamly-shell.git - tag: 070ff21fda7aab8ca45b08b330746299a91eb981 +package bench-report + flags: +no-charts source-repository-package type: git location: https://github.com/composewell/streamly-coreutils.git - tag: 051ef37a5fa399f039d0530e8d642fd79ec6aec2 + tag: fa180060c7510c89d2767980ca6f7ec7011d04b9 source-repository-package type: git location: https://github.com/composewell/bench-report.git - tag: a9bbaecb19eda832edf32ae274fef48895facf0b + tag: 4dad3ea916a950524bcfd5097fc6f0f63e645987 -package bench-report - flags: +no-charts +-- source-repository-package +-- type: git +-- location: https://github.com/composewell/streamly.git +-- tag: 9a039b89c6b81300b2bd9c8765bf4bfa18e7c83c +-- +-- source-repository-package +-- type: git +-- location: https://github.com/composewell/streamly.git +-- tag: 9a039b89c6b81300b2bd9c8765bf4bfa18e7c83c +-- subdir: core +-- +-- source-repository-package +-- type: git +-- location: https://github.com/composewell/streamly-process.git +-- tag: c1ce40ebe84973d0c54dd27fb17e337bf9ca4a46 +-- +-- source-repository-package +-- type: git +-- location: https://github.com/composewell/streamly-shell.git +-- tag: 070ff21fda7aab8ca45b08b330746299a91eb981 diff --git a/default.nix b/default.nix deleted file mode 100644 index 60cce9e..0000000 --- a/default.nix +++ /dev/null @@ -1,102 +0,0 @@ -# CAUTION! a spelling mistake in arg string is ignored silently. -# -# nix-shell --argstr c2nix "--flag examples-sdl" - -# To use ghc-8.6.5 -# nix-shell --argstr compiler "ghc865" - -{ - nixpkgs ? - import (builtins.fetchTarball - https://github.com/NixOS/nixpkgs/archive/refs/tags/22.05.tar.gz) - {} -, compiler ? "ghc922" -, c2nix ? "" # cabal2nix CLI options -# TODO -#, sources ? [] # e.g. [./. ./benchmark] -#, hdeps ? [] # e.g. [time, mtl] -#, deps ? [] # e.g. [SDL2] -}: -let haskellPackages = - if compiler == "default" - then nixpkgs.haskellPackages - else nixpkgs.haskell.packages.${compiler}; - - # we can possibly avoid adding our package to HaskellPackages like - # in the case of nix-shell for a single package? - mkPackage = super: pkg: path: opts: inShell: - let orig = super.callCabal2nixWithOptions pkg path opts {}; - in if inShell - # Avoid copying the source directory to nix store by using - # src = null. - then orig.overrideAttrs (oldAttrs: { src = null; }) - else orig; - - flags = "--benchmark" + " " + c2nix; - - mkHaskellPackages = inShell: - haskellPackages.override { - # We could disbale doCheck on all like this, but it would make the - # whole world rebuild, we can't use the binary cache - #packageSetConfig = self: super: { - # mkDerivation = drv: super.mkDerivation (drv // { - # doCheck = false; - # }); - #}; - overrides = self: super: - with nixpkgs.haskell.lib; - { - streaming-benchmarks = - mkPackage super "streaming-benchmarks" - ./. flags inShell; - - streamly = - nixpkgs.haskell.lib.overrideCabal - (super.callHackageDirect - { pkg = "streamly"; - ver = "0.8.3"; - sha256 = "sha256-A6/S/yvmOXVabVAR6x4a/XYszzeogSU4vOuyLLSGFvE="; - } {}) - (old: - { librarySystemDepends = - if builtins.currentSystem == "x86_64-darwin" - then [nixpkgs.darwin.apple_sdk.frameworks.Cocoa] - else []; - enableLibraryProfiling = false; - doHaddock = false; - }); - - lockfree-queue = - super.callHackageDirect - { pkg = "lockfree-queue"; - ver = "0.2.4"; - sha256 = "1bj9agy3x0yjbscpjgn96gpnj4lvkh39spjvy3jnrr3a42v3ynw7"; - } {}; - - #fusion-plugin = - # super.callHackageDirect - # { pkg = "fusion-plugin"; - # ver = "0.2.3"; - # sha256 = "073wbhdxj1sh5160blaihbzkkhabs8s71pqhag16lvmgbb7a3hla"; - # } {}; - }; - }; - - hspkgs = mkHaskellPackages true; - - shell = hspkgs.shellFor { - packages = p: - [ p.streaming-benchmarks - ]; - doBenchmark = true; - shellHook = '' - export CABAL_DIR="$(pwd)/.cabal.nix" - if test -n "$PS_SHELL" - then - export PS1="$PS_SHELL\[$bldred\](nix)\[$txtrst\] " - fi - ''; - }; -in if nixpkgs.lib.inNixShell - then shell - else (mkHaskellPackages false).streaming-benchmarks diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..d0d6dbb --- /dev/null +++ b/flake.lock @@ -0,0 +1,88 @@ +{ + "nodes": { + "basepkgs": { + "inputs": { + "basepkgs": "basepkgs_2", + "nixpkgs": "nixpkgs", + "nixpkgs-darwin": "nixpkgs-darwin" + }, + "locked": { + "lastModified": 1774694867, + "narHash": "sha256-Cgch0+tWHddjTUMhXCiUpu4yDGNFdMb9BAtyjv7+mTw=", + "ref": "refs/heads/master", + "rev": "0f2afe624cb0e259edbc87bdf60aabdd4cf2580a", + "revCount": 117, + "type": "git", + "url": "ssh://git@github.com/composewell/streamly-packages" + }, + "original": { + "rev": "0f2afe624cb0e259edbc87bdf60aabdd4cf2580a", + "type": "git", + "url": "ssh://git@github.com/composewell/streamly-packages" + } + }, + "basepkgs_2": { + "locked": { + "lastModified": 1774694658, + "narHash": "sha256-E4PwUiMKxH7paTq/6KCkudmJDTICOV4PdLOVxlWclwQ=", + "owner": "composewell", + "repo": "nixpack", + "rev": "57811ee9a97c48b5ec84909bc137eb57361651b0", + "type": "github" + }, + "original": { + "owner": "composewell", + "repo": "nixpack", + "rev": "57811ee9a97c48b5ec84909bc137eb57361651b0", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1774106199, + "narHash": "sha256-US5Tda2sKmjrg2lNHQL3jRQ6p96cgfWh3J1QBliQ8Ws=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6c9a78c09ff4d6c21d0319114873508a6ec01655", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6c9a78c09ff4d6c21d0319114873508a6ec01655", + "type": "github" + } + }, + "nixpkgs-darwin": { + "locked": { + "lastModified": 1774106199, + "narHash": "sha256-US5Tda2sKmjrg2lNHQL3jRQ6p96cgfWh3J1QBliQ8Ws=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6c9a78c09ff4d6c21d0319114873508a6ec01655", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6c9a78c09ff4d6c21d0319114873508a6ec01655", + "type": "github" + } + }, + "root": { + "inputs": { + "basepkgs": "basepkgs", + "nixpkgs": [ + "basepkgs", + "nixpkgs" + ], + "nixpkgs-darwin": [ + "basepkgs", + "nixpkgs-darwin" + ] + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..f5c8ebe --- /dev/null +++ b/flake.nix @@ -0,0 +1,17 @@ +{ + description = "Streaming-benchmarks"; + + inputs = { + basepkgs.url = "git+ssh://git@github.com/composewell/streamly-packages?rev=0f2afe624cb0e259edbc87bdf60aabdd4cf2580a"; + nixpkgs.follows = "basepkgs/nixpkgs"; + nixpkgs-darwin.follows = "basepkgs/nixpkgs-darwin"; + }; + + outputs = { self, nixpkgs, nixpkgs-darwin, basepkgs }: + basepkgs.nixpack.mkOutputs { + inherit nixpkgs nixpkgs-darwin basepkgs; + name = "streaming-benchmarks"; + sources = import ./sources.nix; + packages = import ./packages.nix; + }; +} diff --git a/packages.nix b/packages.nix new file mode 100644 index 0000000..642a684 --- /dev/null +++ b/packages.nix @@ -0,0 +1,24 @@ +{ nixpkgs }: +{ +# General packages, haskell or non-haskell will be installed in the +# environment. +packages = + with nixpkgs.pkgs; + [ + ]; + +# Haskell packages available in the nix shell, the dependencies of these +# packages are also installed and exposed in ghc package database. +libraries = +with nixpkgs.haskellPackages; +[ + fusion-plugin streamly-core bench-report vector dlist +]; + +# Haskell dev packages. Install only dependencies of these in the shell, +# do not build the packages themselves. +dev-packages = + with nixpkgs.haskellPackages; + [ streaming-benchmarks ]; + +} diff --git a/sources.nix b/sources.nix new file mode 100644 index 0000000..077ad08 --- /dev/null +++ b/sources.nix @@ -0,0 +1,12 @@ +{nixpack}: +with nixpack.mkSources; +{ +layers = [ +{ + streaming-benchmarks = local ./.; + #streaming-benchmarks = localOpts ./. + #[] + #["--flags streaming"]; +} +]; +} diff --git a/stack.yaml b/stack.yaml deleted file mode 100644 index 7caba7f..0000000 --- a/stack.yaml +++ /dev/null @@ -1,9 +0,0 @@ -resolver: lts-23.27 -packages: -- '.' - -extra-deps: - - streamly-0.11.0 - - streamly-core-0.3.0 - -rebuild-ghc-options: true diff --git a/streaming-benchmarks.cabal b/streaming-benchmarks.cabal index 2fef636..121e658 100644 --- a/streaming-benchmarks.cabal +++ b/streaming-benchmarks.cabal @@ -25,7 +25,16 @@ description: * conduit * drinkery -tested-with: GHC==8.8.4, GHC==8.10.4 +tested-with: + GHC==8.10.7 + , GHC==9.2.8 + , GHC==9.4.4 + , GHC==9.6.3 + , GHC==9.8.1 + , GHC==9.10.3 + , GHC==9.12.1 + , GHC==9.14.1 + build-type: Simple extra-source-files: Benchmarks/VectorCommon.hs @@ -97,11 +106,11 @@ common lib-options mtl >= 2.2.2 && < 2.4, random >= 1.0 && < 2.0, transformers >= 0.5.5 && < 0.7, - template-haskell >= 2.14 && < 2.24 + template-haskell >= 2.14 && < 2.25 if flag(use-gauge) build-depends: gauge >= 0.2.4 && < 0.3 else - build-depends: tasty-bench >= 0.3 && < 0.5 + build-depends: tasty-bench >= 0.3 && < 0.6 , tasty >= 1.4.1 && < 1.6 mixins: tasty-bench (Test.Tasty.Bench as Gauge