collab/1brc

the one billion row challenge

git clone https://git.julia.blue/collab/1brc

Rahul Butanimisc: init0d0166a

back-to-mmap
4.0 KiB83 linesraw
1diff --git a/CreateMeasurements3.java b/CreateMeasurements3.java
2index 9bcc16d..9b8df75 100644
3--- a/CreateMeasurements3.java
4+++ b/CreateMeasurements3.java
5@@ -13,7 +13,6 @@
6  *  See the License for the specific language governing permissions and
7  *  limitations under the License.
8  */
9-package dev.morling.onebrc;
10 
11 import java.io.BufferedReader;
12 import java.io.BufferedWriter;
13@@ -24,7 +23,7 @@ import java.io.StringReader;
14 import java.nio.charset.StandardCharsets;
15 import java.util.ArrayList;
16 import java.util.HashSet;
17-import java.util.concurrent.ThreadLocalRandom;
18+import java.util.Random;
19 
20 public class CreateMeasurements3 {
21 
22@@ -32,23 +31,26 @@ public class CreateMeasurements3 {
23     public static final int KEYSET_SIZE = 10_000;
24 
25     public static void main(String[] args) throws Exception {
26-        if (args.length != 1) {
27-            System.out.println("Usage: create_measurements3.sh <number of records to create>");
28+        if (args.length != 3) {
29+            System.out.println("Usage: create_measurements3.sh <weather stations csv path> <number of records to create> <seed>");
30             System.exit(1);
31         }
32+        String weatherStationsCsv = args[0];
33         int size = 0;
34+        int seed = 0;
35         try {
36-            size = Integer.parseInt(args[0]);
37+            size = Integer.parseInt(args[1]);
38+            seed = Integer.parseInt(args[2]);
39         }
40         catch (NumberFormatException e) {
41-            System.out.println("Invalid value for <number of records to create>");
42-            System.out.println("Usage: create_measurements3.sh <number of records to create>");
43+            System.out.println("Invalid value for <number of records to create> or <seed>");
44+            System.out.println("Usage: create_measurements3.sh <weather stations csv path> <number of records to create> <seed>");
45             System.exit(1);
46         }
47-        final var weatherStations = generateWeatherStations();
48+        final var rnd = new Random(seed);
49+        final var weatherStations = generateWeatherStations(rnd, weatherStationsCsv);
50         final var start = System.currentTimeMillis();
51-        final var rnd = ThreadLocalRandom.current();
52-        try (var out = new BufferedWriter(new FileWriter("measurements3.txt"))) {
53+        try (var out = new BufferedWriter(new FileWriter("measurements.txt"))) {
54             for (int i = 1; i <= size; i++) {
55                 var station = weatherStations.get(rnd.nextInt(KEYSET_SIZE));
56                 double temp = rnd.nextGaussian(station.avgTemp, 7.0);
57@@ -66,11 +68,11 @@ public class CreateMeasurements3 {
58     record WeatherStation(String name, float avgTemp) {
59     }
60 
61-    private static ArrayList<WeatherStation> generateWeatherStations() throws Exception {
62+    private static ArrayList<WeatherStation> generateWeatherStations(Random rnd, String weatherStationsCsvPath) throws Exception {
63         // Use a public list of city names and concatenate them all into a long string,
64         // which we'll use as a "source of city name randomness"
65         var bigName = new StringBuilder(1 << 20);
66-        try (var rows = new BufferedReader(new FileReader("data/weather_stations.csv"));) {
67+        try (var rows = new BufferedReader(new FileReader(weatherStationsCsvPath));) {
68             skipComments(rows);
69             while (true) {
70                 var row = rows.readLine();
71@@ -84,11 +86,10 @@ public class CreateMeasurements3 {
72         final var names = new HashSet<String>();
73         var minLen = Integer.MAX_VALUE;
74         var maxLen = Integer.MIN_VALUE;
75-        try (var rows = new BufferedReader(new FileReader("data/weather_stations.csv"))) {
76+        try (var rows = new BufferedReader(new FileReader(weatherStationsCsvPath))) {
77             skipComments(rows);
78             final var nameSource = new StringReader(bigName.toString());
79             final var buf = new char[MAX_NAME_LEN];
80-            final var rnd = ThreadLocalRandom.current();
81             final double yOffset = 4;
82             final double factor = 2500;
83             final double xOffset = 0.372;