{ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.flake-utils.url = "github:numtide/flake-utils"; inputs.rust-overlay.url = "github:oxalica/rust-overlay"; inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; outputs = inputs: inputs.flake-utils.lib.eachDefaultSystem (system: let pkgs = inputs.nixpkgs.legacyPackages.${system}.extend ( inputs.rust-overlay.overlays.default ); inherit (inputs.nixpkgs) lib; upstream-1brc = { stdenv , lib , writeShellApplication , makeWrapper , fetchFromGitHub , javaPackages , graalvmPackages }: let openjdk = javaPackages.compiler.openjdk25; in stdenv.mkDerivation (finalAttrs: { name = "1brc"; version = finalAttrs.src.rev; src = fetchFromGitHub { owner = "gunnarmorling"; repo = "1brc"; rev = "db064194be375edc02d6dbcd21268ad40f7e2869"; hash = "sha256-SveG6OAFGYNA6IgmTS1r+bUBgVQwamy9lE+mwZr70Wg="; }; # generators: # - not bothering to pre-compile # - using `CreateMeasurements` instead of the faster # `CreateMeasurements{2,Fast}` because it is deterministic (enabling # us to cache it) defaultGenerator = writeShellApplication { name = "generator"; runtimeInputs = [ openjdk ]; derivationArgs.preCheck = '' mkdir -p $out/misc/ cp ${finalAttrs.src}/src/main/java/dev/morling/onebrc/CreateMeasurements.java $out/misc (cd $out/misc; patch < ${./misc/00-gen-rand-seed.patch}) ''; text = '' # $1: output dir, $2: num output rows, $3: random seed cd "$1" src="${builtins.placeholder "out"}/misc/CreateMeasurements.java" java "$src" "$2" "$3" ''; }; manyStationsGenerator = writeShellApplication { name = "many-stations-generator"; runtimeInputs = [ openjdk ]; derivationArgs.preCheck = '' mkdir -p $out/misc cp ${finalAttrs.src}/src/main/java/dev/morling/onebrc/CreateMeasurements3.java $out/misc cp ${finalAttrs.src}/data/weather_stations.csv $out/misc (cd $out/misc; patch < ${./misc/01-gen-many-stations-rand-seed.patch}) ''; text = '' # $1: output dir, $2: num output rows, $3: random seed cd "$1" src="${builtins.placeholder "out"}/misc/CreateMeasurements3.java" csv="${builtins.placeholder "out"}/misc/weather_stations.csv" java "$src" "$csv" "$2" "$3" ''; }; baselineImpl = stdenv.mkDerivation (this: { name = "1brc-baseline"; inherit (finalAttrs) version; dontUnpack = true; nativeBuildInputs = [ openjdk makeWrapper ]; buildPhase = '' src="${finalAttrs.src}/src/main/java/dev/morling/onebrc/CalculateAverage_baseline.java" mkdir -p "$out" javac -d "$out/" "$src" ''; installPhase = '' mkdir -p "$out/bin" makeWrapper "${lib.getExe' openjdk "java"}" "$out/bin/$name" \ --add-flag "-Dsun.stdout.encoding=UTF-8" \ --add-flag "-cp" \ --add-flag "$out" \ --add-flag "dev.morling.onebrc.CalculateAverage_baseline" ''; meta.mainProgram = this.name; passthru.commandString = builtins.concatStringsSep " " [ "${lib.getExe' openjdk "java"}" "-Dsun.stdout.encoding=UTF-8" "-cp ${this.finalPackage.outPath}" "dev.morling.onebrc.CalculateAverage_baseline" ]; }); thomaswueImpl = { arch ? "x86-64-v4" }: stdenv.mkDerivation (this: { name = "1brc-thomaswue"; inherit (finalAttrs) version; dontUnpack = true; nativeBuildInputs = [ openjdk graalvmPackages.graalvm-ce ]; # see: `${finalAttrs.src}/prepare_thomaswue.sh graalCompileOpts = [ "-O3" # "-H:TuneInlinerExploration=1" "-march=${arch}" "--enable-preview" "--initialize-at-build-time=dev.morling.onebrc.CalculateAverage_thomaswue\$Scanner" "--gc=epsilon" "-H:-GenLoopSafepoints" ]; buildPhase = '' src="${finalAttrs.src}/src/main/java/dev/morling/onebrc/CalculateAverage_thomaswue.java" javac -d . "$src" mkdir -p "$out/bin/" native-image $graalCompileOpts -o "$out/bin/$name" dev.morling.onebrc.CalculateAverage_thomaswue ''; meta.mainProgram = this.name; passthru.commandString = lib.getExe this.finalPackage; }); dontUnpack = true; installPhase = '' mkdir -p $out/bin ln -s ${finalAttrs.defaultGenerator} $out/bin/ ln -s ${finalAttrs.manyStationsGenerator} $out/bin/ ''; }); materializeInput = name: command: outVar: let dirPath = "/tmp/_1br/${name}"; in '' if ! [[ -f "${dirPath}/measurements.txt" ]]; then mkdir -p "${dirPath}" (cd "${dirPath}"; ${command}) fi ${outVar}="${dirPath}" ''; in { legacyPackages."1brc" = pkgs.callPackage upstream-1brc {}; # don't want to put inputs in the nix store (large) so we generate them as # needed; results are small and fine to cache # - generateInputsCommand: produces `measurements.txt` in cwd # - expectedOutput: path to a file containing the expected output legacyPackages.data = let onebrc = inputs.self.legacyPackages.${system}."1brc"; sample = name: let dir = inputs.self.legacyPackages.${system}."1brc".src.outPath; base = dir + "/src/test/resources/samples/measurements-"; in { generateInputCommand = let path = base + name + ".txt"; in "${lib.getExe' pkgs.coreutils "cp"} ${path} measurements.txt"; expectedOutput = base + name + ".out"; }; # generator: executable that takes: ` ` generated = generator: seed: rows: rec { generateInputCommand = "${lib.getExe generator} . ${toString rows} ${toString seed}"; expectedOutput = let name = "${generator.name}-${toString seed}-rows${toString rows}.out"; in pkgs.runCommand name {} '' ${generateInputCommand} ${lib.getExe onebrc.baselineImpl} > $out ''; }; genDefault = generated onebrc.defaultGenerator; genManySta = generated onebrc.manyStationsGenerator; in { sample-1 = sample "1"; sample-10 = sample "10"; sample-10000-unique-keys = sample "10000-unique-keys"; sample-2 = sample "2"; sample-20 = sample "20"; sample-3 = sample "3"; sample-boundaries = sample "boundaries"; sample-complex-utf8 = sample "complex-utf8"; sample-dot = sample "dot"; sample-rounding = sample "rounding"; sample-short = sample "short"; sample-shortest = sample "shortest"; small = genDefault 0 (100 * 1000); "1br" = genDefault 0 (1000 * 1000 * 1000); "1br-many-stations" = genManySta 0 (1000 * 1000 * 1000); }; legacyPackages.testRunners = let testFunc = name: { generateInputCommand, expectedOutput }: pkgs.writeShellApplication { name = "test-${name}"; runtimeInputs = [ pkgs.coreutils pkgs.delta ]; text = '' # $@: command to run the implementation set -- "$(realpath "$1")" "''${@:2}" outFile="$(realpath "$(mktemp)")" ${materializeInput name generateInputCommand "inputDir"} (cd "$inputDir"; "$@" > "$outFile") delta -s \ <(sed 's|, |\n|g' < "$outFile") \ <(sed 's|, |\n|g' < "${expectedOutput}") rm "$outFile" ''; }; perData = builtins.mapAttrs testFunc inputs.self.legacyPackages.${system}.data; all = pkgs.writeShellApplication { name = "test-all"; text = let forTest = name: pkg: '' # shellcheck disable=SC2015 ${lib.getExe pkg} "$@" && echo "PASS: ${name}" || { echo "FAIL: ${name}"; exit 1 } ''; testCmds = lib.pipe perData [ (lib.mapAttrsToList forTest) (builtins.concatStringsSep "\n") ]; in '' # $@: command to run the implementation ${testCmds} ''; }; in perData // { inherit all; } // all; legacyPackages.benchRunners = let benchFunc = name: { generateInputCommand, ... }: pkgs.writeShellApplication { name = "bench-${name}"; runtimeInputs = [ pkgs.hyperfine ]; text = '' # $@: command to run the implementation # $HYPERFINE_OPTS: extra flags to hyperfine set -- "$(realpath "$1")" "''${@:2}" echo "BENCH: ${name}" ${materializeInput name generateInputCommand "inputDir"} cd "$inputDir" # shellcheck disable=SC2086 hyperfine \ --shell=none \ --time-unit=millisecond \ ''${HYPERFINE_OPTS-} \ "$*" ''; }; in builtins.mapAttrs benchFunc inputs.self.legacyPackages.${system}.data; legacyPackages.impls = let onebrc = inputs.self.legacyPackages.${system}."1brc"; in { baseline = onebrc.baselineImpl; "1stAvx10" = onebrc.thomaswueImpl { arch = "x86-64-v4"; }; "1stAvx2" = onebrc.thomaswueImpl { arch = "x86-64-v3"; }; # this; # TODO }; legacyPackages.bench = let impls = inputs.self.legacyPackages.${system}.impls; benchRunners = inputs.self.legacyPackages.${system}.benchRunners; mkBench = benchRunner: impl: pkgs.writeShellApplication { name = "${benchRunner.name}-${impl.name}"; text = '' ${lib.getExe benchRunner} ${impl.commandString} ''; }; in lib.pipe impls [ (builtins.mapAttrs (_: impl: lib.pipe benchRunners [ (builtins.mapAttrs (_: benchRunner: mkBench benchRunner impl)) # lib.recurseIntoAttrs ])) # lib.recurseIntoAttrs ]; checks = let test-all = inputs.self.legacyPackages.${system}.testRunners.all; onebrc = inputs.self.legacyPackages.${system}."1brc"; mkCheck = name: pkg: { ${name} = pkgs.runCommand "${name}-test" {} '' ${lib.getExe test-all} ${pkg.commandString} && touch $out ''; }; in (mkCheck "baseline" onebrc.baselineImpl) // (mkCheck "thomaswueAvx2" (onebrc.thomaswueImpl { arch = "x86-64-v3"; })) # TODO: this ; # aliases, for convenience: apps = let mkApp = pkg: { type = "app"; program = lib.getExe pkg; }; this = inputs.self.legacyPackages.${system}; in { testRunner = mkApp this.testRunners.all; benchRunner = mkApp this.benchRunners."1br"; benchRunnerSmall = mkApp this.benchRunners.small; benchRunnerManyStations = mkApp this.benchRunners."1br-many-stations"; bench-baseline = mkApp this.bench.baseline."1br"; bench-1st-avx2 = mkApp this.bench."1stAvx2"."1br"; bench-1st-avx10 = mkApp this.bench."1stAvx10"."1br"; benchSmall-baseline = mkApp this.bench.baseline.small; benchSmall-1st-avx2 = mkApp this.bench."1stAvx2".small; benchSmall-1st-avx10 = mkApp this.bench."1stAvx10".small; # todo: test, bench (for packages.default) }; # packages.default; # TODO devShells.default = pkgs.mkShell { packages = with pkgs; let this = inputs.self.legacyPackages.${system}; in [ hyperfine this.testRunners.all this.benchRunners.small this.benchRunners."1br" this.benchRunners."1br-many-stations" this.impls.baseline ] ++ lib.optionals pkgs.stdenv.isLinux (with pkgs; [ # linuxKernel.packages.linux_6_12.perf # will vary perf # todo: perf-tools? # this.impls."1stAvx10" this.impls."1stAvx2" ]); # env. }; }); }