{ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.flake-utils.url = "github:numtide/flake-utils"; inputs.rust-overlay.url = "github:oxalica/rust-overlay"; inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; outputs = inputs: inputs.flake-utils.lib.eachDefaultSystem (system: let pkgs = inputs.nixpkgs.legacyPackages.${system}.extend ( inputs.rust-overlay.overlays.default ); inherit (inputs.nixpkgs) lib; rust-toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; upstream-1brc = { stdenv , lib , writeShellApplication , makeWrapper , fetchFromGitHub , javaPackages , graalvmPackages }: let openjdk = javaPackages.compiler.openjdk25; in stdenv.mkDerivation (finalAttrs: { name = "1brc"; version = finalAttrs.src.rev; src = fetchFromGitHub { owner = "gunnarmorling"; repo = "1brc"; rev = "db064194be375edc02d6dbcd21268ad40f7e2869"; hash = "sha256-SveG6OAFGYNA6IgmTS1r+bUBgVQwamy9lE+mwZr70Wg="; }; # generators: # - not bothering to pre-compile # - using `CreateMeasurements` instead of the faster # `CreateMeasurements{2,Fast}` because it is deterministic (enabling # us to cache it) defaultGenerator = writeShellApplication { name = "generator"; runtimeInputs = [ openjdk ]; derivationArgs.preCheck = '' mkdir -p $out/misc/ cp ${finalAttrs.src}/src/main/java/dev/morling/onebrc/CreateMeasurements.java $out/misc (cd $out/misc; patch < ${./misc/00-gen-rand-seed.patch}) ''; text = '' # $1: output dir, $2: num output rows, $3: random seed cd "$1" src="${builtins.placeholder "out"}/misc/CreateMeasurements.java" java "$src" "$2" "$3" ''; }; manyStationsGenerator = writeShellApplication { name = "many-stations-generator"; runtimeInputs = [ openjdk ]; derivationArgs.preCheck = '' mkdir -p $out/misc cp ${finalAttrs.src}/src/main/java/dev/morling/onebrc/CreateMeasurements3.java $out/misc cp ${finalAttrs.src}/data/weather_stations.csv $out/misc (cd $out/misc; patch < ${./misc/01-gen-many-stations-rand-seed.patch}) ''; text = '' # $1: output dir, $2: num output rows, $3: random seed cd "$1" src="${builtins.placeholder "out"}/misc/CreateMeasurements3.java" csv="${builtins.placeholder "out"}/misc/weather_stations.csv" java "$src" "$csv" "$2" "$3" ''; }; baselineImpl = stdenv.mkDerivation (this: { pname = "1brc-baseline"; inherit (finalAttrs) version; dontUnpack = true; nativeBuildInputs = [ openjdk makeWrapper ]; buildPhase = '' src="${finalAttrs.src}/src/main/java/dev/morling/onebrc/CalculateAverage_baseline.java" mkdir -p "$out" javac -d "$out/" "$src" ''; installPhase = '' mkdir -p "$out/bin" makeWrapper "${lib.getExe' openjdk "java"}" "$out/bin/$pname" \ --add-flag "-Dsun.stdout.encoding=UTF-8" \ --add-flag "-cp" \ --add-flag "$out" \ --add-flag "dev.morling.onebrc.CalculateAverage_baseline" ''; meta.mainProgram = this.pname; passthru.commandString = builtins.concatStringsSep " " [ "${lib.getExe' openjdk "java"}" "-Dsun.stdout.encoding=UTF-8" "-cp ${this.finalPackage.outPath}" "dev.morling.onebrc.CalculateAverage_baseline" ]; }); thomaswueImpl = { arch ? "x86-64-v4", extraAttrs ? {} }: stdenv.mkDerivation (this: { pname = "1brc-thomaswue-${arch}"; inherit (finalAttrs) version; dontUnpack = true; nativeBuildInputs = [ openjdk graalvmPackages.graalvm-ce ]; # see: `${finalAttrs.src}/prepare_thomaswue.sh graalCompileOpts = [ "-O3" # "-H:TuneInlinerExploration=1" "-march=${arch}" "--enable-preview" "--initialize-at-build-time=dev.morling.onebrc.CalculateAverage_thomaswue\$Scanner" "--gc=epsilon" "-H:-GenLoopSafepoints" "-H:-CheckToolchain" ]; buildPhase = '' src="${finalAttrs.src}/src/main/java/dev/morling/onebrc/CalculateAverage_thomaswue.java" javac -d . "$src" mkdir -p "$out/bin/" native-image $graalCompileOpts -o "$out/bin/$pname" dev.morling.onebrc.CalculateAverage_thomaswue ''; meta.mainProgram = this.pname; passthru.commandString = lib.getExe this.finalPackage; } // extraAttrs); dontUnpack = true; installPhase = '' mkdir -p $out/bin ln -s ${finalAttrs.defaultGenerator} $out/bin/ ln -s ${finalAttrs.manyStationsGenerator} $out/bin/ ''; }); materializeInput = name: command: outVar: '' _dirPath="''${ONEBRC_TEST_DIR-"/tmp/_1br"}/${name}" if ! [[ -f "''${_dirPath}/measurements.txt" ]]; then mkdir -p "''${_dirPath}" (cd "''${_dirPath}"; ${command}) fi ${outVar}="''${_dirPath}" ''; plat = pkgs.stdenv.hostPlatform; platformsFor1st = { # name => { arch: graal target arch, extraAttrs: {} } Native = { arch = "native"; # need to be careful not to upload to substituters... extraAttrs = { allowSubstitutes = false; preferLocalBuild = true; }; }; } // lib.optionalAttrs plat.isx86_64 { Avx2.arch = "x86-64-v3"; Avx10.arch = "x86-64-v4"; } // lib.optionalAttrs plat.isAarch64 { # all: FP + ASIMD + EVTSTRM, + AES + PMULL + SHA1 + SHA2 + CRC32 + LSE + DCPOP + SHA3 + SHA512 + SVE + PACA + SVEBITPERM + SVE2 + A53MAC # armv8-a: FP + ASIMD # armv8.1-a: FP + ASIMD + + CRC32 + LSE # m1: FP + ASIMD + AES + PMULL + SHA1 + SHA2 + CRC32 + LSE + SHA3 + SHA512 # m4: FP + ASIMD + AES + PMULL + SHA1 + SHA2 + CRC32 + LSE + SHA3 + SHA512 # # unfortunately, can only enable `aes`, `lse`, `fp`, `simd` via feature modifiers... ASIMD.arch = "armv8.1-a+aes"; }; in { legacyPackages."1brc" = pkgs.callPackage upstream-1brc {}; # don't want to put inputs in the nix store (large) so we generate them as # needed; results are small and fine to cache # - generateInputsCommand: produces `measurements.txt` in cwd # - expectedOutput: path to a file containing the expected output legacyPackages.data = let onebrc = inputs.self.legacyPackages.${system}."1brc"; sample = name: let dir = inputs.self.legacyPackages.${system}."1brc".src.outPath; base = dir + "/src/test/resources/samples/measurements-"; in { generateInputCommand = let path = base + name + ".txt"; in "${lib.getExe' pkgs.coreutils "cp"} ${path} measurements.txt"; expectedOutput = base + name + ".out"; }; # generator: executable that takes: ` ` generated = generator: seed: rows: rec { generateInputCommand = "${lib.getExe generator} . ${toString rows} ${toString seed}"; expectedOutput = let name = "${generator.name}-${toString seed}-rows${toString rows}.out"; in pkgs.runCommand name {} '' ${generateInputCommand} ${lib.getExe onebrc.baselineImpl} > $out ''; }; genDefault = generated onebrc.defaultGenerator; genManySta = generated onebrc.manyStationsGenerator; in { sample-1 = sample "1"; sample-10 = sample "10"; sample-10000-unique-keys = sample "10000-unique-keys"; sample-2 = sample "2"; sample-20 = sample "20"; sample-3 = sample "3"; sample-boundaries = sample "boundaries"; sample-complex-utf8 = sample "complex-utf8"; sample-dot = sample "dot"; sample-rounding = sample "rounding"; sample-short = sample "short"; sample-shortest = sample "shortest"; small = genDefault 0 (100 * 1000); "1br" = genDefault 0 (1000 * 1000 * 1000); "1br-many-stations" = genManySta 0 (1000 * 1000 * 1000); }; legacyPackages.testRunners = let testFunc = name: { generateInputCommand, expectedOutput }: pkgs.writeShellApplication { name = "test-${name}"; runtimeInputs = [ pkgs.coreutils pkgs.delta ]; text = '' # $@: command to run the implementation set -- "$(realpath "$1")" "''${@:2}" outFile="$(realpath "$(mktemp)")" ${materializeInput name generateInputCommand "inputDir"} (cd "$inputDir"; "$@" > "$outFile") delta -s \ <(sed 's|, |\n|g' < "$outFile") \ <(sed 's|, |\n|g' < "${expectedOutput}") rm "$outFile" ''; }; perData = builtins.mapAttrs testFunc inputs.self.legacyPackages.${system}.data; all = pkgs.writeShellApplication { name = "test-all"; text = let forTest = name: pkg: '' # shellcheck disable=SC2015 ${lib.getExe pkg} "$@" && echo "PASS: ${name}" || { echo "FAIL: ${name}"; exit 1 } ''; testCmds = lib.pipe perData [ (lib.mapAttrsToList forTest) (builtins.concatStringsSep "\n") ]; in '' # $@: command to run the implementation ${testCmds} ''; }; in all // perData // { inherit all; }; legacyPackages.benchRunners = let benchFunc = name: { generateInputCommand, ... }: pkgs.writeShellApplication { name = "bench-${name}"; runtimeInputs = [ pkgs.hyperfine ]; text = '' # $@: command to run the implementation # $HYPERFINE_OPTS: extra flags to hyperfine set -- "$(realpath "$1")" "''${@:2}" echo "BENCH: ${name}" ${materializeInput name generateInputCommand "inputDir"} cd "$inputDir" # shellcheck disable=SC2086 hyperfine \ --shell=none \ --time-unit=millisecond \ ''${HYPERFINE_OPTS-} \ "$*" ''; }; in builtins.mapAttrs benchFunc inputs.self.legacyPackages.${system}.data; legacyPackages.impls = let onebrc = inputs.self.legacyPackages.${system}."1brc"; in { baseline = onebrc.baselineImpl; # this; # TODO } // lib.mapAttrs' (name: attrs: { name = "1st${name}"; value = onebrc.thomaswueImpl attrs; }) platformsFor1st; legacyPackages.bench = let impls = inputs.self.legacyPackages.${system}.impls; benchRunners = inputs.self.legacyPackages.${system}.benchRunners; mkBench = benchRunner: impl: pkgs.writeShellApplication { name = "${benchRunner.name}-${impl.pname or impl.name}"; text = '' ${lib.getExe benchRunner} ${impl.commandString} ''; }; in lib.pipe impls [ (builtins.mapAttrs (_: impl: lib.pipe benchRunners [ (builtins.mapAttrs (_: benchRunner: mkBench benchRunner impl)) # lib.recurseIntoAttrs ])) # lib.recurseIntoAttrs ]; checks = let impls = inputs.self.legacyPackages.${system}.impls; test-all = inputs.self.legacyPackages.${system}.testRunners.all; onebrc = inputs.self.legacyPackages.${system}."1brc"; mkCheck = name: pkg: pkgs.runCommand "${name}-test" {} '' export ONEBRC_TEST_DIR=$(realpath ./test-data) ${lib.getExe test-all} ${pkg.commandString} && touch $out ''; in lib.mapAttrs' (name': impl: rec { name = builtins.replaceStrings ["1st"] ["thomaswue"] name'; value = mkCheck name impl; }) impls ; # aliases, for convenience: apps = let mkApp = pkg: { type = "app"; program = lib.getExe pkg; }; this = inputs.self.legacyPackages.${system}; in { testRunner = mkApp this.testRunners.all; benchRunner = mkApp this.benchRunners."1br"; benchRunnerSmall = mkApp this.benchRunners.small; benchRunnerManyStations = mkApp this.benchRunners."1br-many-stations"; bench-baseline = mkApp this.bench.baseline."1br"; benchSmall-baseline = mkApp this.bench.baseline.small; # todo: test, bench (for packages.default) } // lib.mapAttrs' (name': _: { name = "bench-1st-${name'}"; value = mkApp this.bench."1st${name'}"."1br"; }) platformsFor1st // lib.mapAttrs' (name': _: { name = "benchSmall-1st-${name'}"; value = mkApp this.bench."1st${name'}".small; }) platformsFor1st ; # packages.default; # TODO devShells.default = pkgs.mkShell { packages = with pkgs; let this = inputs.self.legacyPackages.${system}; in [ rust-toolchain hwloc pkg-config hyperfine vmtouch # todo: lowcharts from julia's config (w/necessary overrides) this.testRunners.all this.benchRunners.small this.benchRunners."1br" this.benchRunners."1br-many-stations" python3 cargo-pgo llvm ] ++ lib.optionals plat.isLinux [ numactl # linuxKernel.packages.linux_6_12.perf # will vary perf # todo: perf-tools? ocamlPackages.magic-trace # needs `legacyPackages.x86_64-linux.linuxPackages_5_15.perf` from nixos-24.11 # MAGIC_TRACE_FZF_DEMANGLE_SYMBOLS # MAGIC_TRACE_PERF_PATH # MAGIC_TRACE_PERF_NO_KCORE — to fix kernel tracing # todo: https://github.com/michoecho/perf2perfetto ] ++ builtins.attrValues this.impls; }; }); }