collab/1brc
the one billion row challenge
git clone https://git.julia.blue/collab/1brc
0d0166a
back-to-mmap
1diff --git a/CreateMeasurements3.java b/CreateMeasurements3.java 2index9bcc16d ..9b8df75 100644 3--- a/CreateMeasurements3.java 4+++ b/CreateMeasurements3.java 5@@ -13,7 +13,6 @@ 6 * See the License for the specific language governing permissions and 7 * limitations under the License. 8 */ 9-package dev.morling.onebrc; 10 11 import java.io.BufferedReader; 12 import java.io.BufferedWriter; 13@@ -24,7 +23,7 @@ import java.io.StringReader; 14 import java.nio.charset.StandardCharsets; 15 import java.util.ArrayList; 16 import java.util.HashSet; 17-import java.util.concurrent.ThreadLocalRandom; 18+import java.util.Random; 19 20 public class CreateMeasurements3 { 21 22@@ -32,23 +31,26 @@ public class CreateMeasurements3 { 23 public static final int KEYSET_SIZE = 10_000; 24 25 public static void main(String[] args) throws Exception { 26- if (args.length != 1) { 27- System.out.println("Usage: create_measurements3.sh <number of records to create>"); 28+ if (args.length != 3) { 29+ System.out.println("Usage: create_measurements3.sh <weather stations csv path> <number of records to create> <seed>"); 30 System.exit(1); 31 } 32+ String weatherStationsCsv = args[0]; 33 int size = 0; 34+ int seed = 0; 35 try { 36- size = Integer.parseInt(args[0]); 37+ size = Integer.parseInt(args[1]); 38+ seed = Integer.parseInt(args[2]); 39 } 40 catch (NumberFormatException e) { 41- System.out.println("Invalid value for <number of records to create>"); 42- System.out.println("Usage: create_measurements3.sh <number of records to create>"); 43+ System.out.println("Invalid value for <number of records to create> or <seed>"); 44+ System.out.println("Usage: create_measurements3.sh <weather stations csv path> <number of records to create> <seed>"); 45 System.exit(1); 46 } 47- final var weatherStations = generateWeatherStations(); 48+ final var rnd = new Random(seed); 49+ final var weatherStations = generateWeatherStations(rnd, weatherStationsCsv); 50 final var start = System.currentTimeMillis(); 51- final var rnd = ThreadLocalRandom.current(); 52- try (var out = new BufferedWriter(new FileWriter("measurements3.txt"))) { 53+ try (var out = new BufferedWriter(new FileWriter("measurements.txt"))) { 54 for (int i = 1; i <= size; i++) { 55 var station = weatherStations.get(rnd.nextInt(KEYSET_SIZE)); 56 double temp = rnd.nextGaussian(station.avgTemp, 7.0); 57@@ -66,11 +68,11 @@ public class CreateMeasurements3 { 58 record WeatherStation(String name, float avgTemp) { 59 } 60 61- private static ArrayList<WeatherStation> generateWeatherStations() throws Exception { 62+ private static ArrayList<WeatherStation> generateWeatherStations(Random rnd, String weatherStationsCsvPath) throws Exception { 63 // Use a public list of city names and concatenate them all into a long string, 64 // which we'll use as a "source of city name randomness" 65 var bigName = new StringBuilder(1 << 20); 66- try (var rows = new BufferedReader(new FileReader("data/weather_stations.csv"));) { 67+ try (var rows = new BufferedReader(new FileReader(weatherStationsCsvPath));) { 68 skipComments(rows); 69 while (true) { 70 var row = rows.readLine(); 71@@ -84,11 +86,10 @@ public class CreateMeasurements3 { 72 final var names = new HashSet<String>(); 73 var minLen = Integer.MAX_VALUE; 74 var maxLen = Integer.MIN_VALUE; 75- try (var rows = new BufferedReader(new FileReader("data/weather_stations.csv"))) { 76+ try (var rows = new BufferedReader(new FileReader(weatherStationsCsvPath))) { 77 skipComments(rows); 78 final var nameSource = new StringReader(bigName.toString()); 79 final var buf = new char[MAX_NAME_LEN]; 80- final var rnd = ThreadLocalRandom.current(); 81 final double yOffset = 4; 82 final double factor = 2500; 83 final double xOffset = 0.372;