collab/1brc
the one billion row challenge
git clone https://git.julia.blue/collab/1brc
dddb811
main
1#!/usr/bin/env python 2 3from collections import Counter ,defaultdict 4import sys 5 6inp = (sys .argv [1 :]or ["/tmp/_1br/1br/measurements.txt" ])[0 ] 7 8def entry_len_dist (path :str )-> Counter [int ]: 9with open (path ,'rb' )as f : 10out = Counter (len (line )for line in f ) 11return out 12 13# num entries in window => (num times seen, Counter of byte len of window) 14typeEntryWindowInfo = dict [int ,tuple [int ,Counter [int ]]] 15def num_entries_in_window (path :str ,window_byte_len :int )-> EntryWindowInfo : 16# naive; doesn't consider chunking... 17out :EntryWindowInfo = defaultdict (lambda : (0 ,Counter ())) 18with open (path ,'rb' )as f : 19curr_len ,count = 0 ,0 20for line in f : 21# add to current window, if fits 22if curr_len + len (line )<= window_byte_len : 23curr_len += len (line ) 24count += 1 25continue 26# otherwise, flush current window and start a new window: 27count_for_window ,byte_counter = out [count ] 28byte_counter .update ([curr_len ]) 29out [count ]= (count_for_window + 1 ,byte_counter ) 30 31curr_len ,count = len (line ),1 32# NOTE: if `len(line)` exceeds `window_byte_len` it'll be flushed 33# individually on the next iteration... but we want to record this 34# separately so denote as zero? 35if curr_len > window_byte_len : 36count = 0 37count_for_window ,byte_counter = out [count ] 38byte_counter .update ([curr_len ]) 39out [count ]= (count_for_window + 1 ,byte_counter ) 40 41curr_len = 0 42 43count_for_window ,byte_counter = out [count ] 44byte_counter .update ([curr_len ]) 45out [count ]= (count_for_window + 1 ,byte_counter ) 46return out 47 48def print_entry_window_info (info :EntryWindowInfo ): 49for num_ents , (seen ,counts )in sorted (info .items ()): 50f" { num_ents :2 } : { seen } " ) 51# print(f" - {counts}") 52avg_byte_len = sum (v * c for v ,c in counts .items ())/ sum (c for _ ,c in counts .items ()) 53f" - { avg_byte_len } " ) 54 55sorted (entry_len_dist (inp ).items ())) 5616 ) 58print_entry_window_info (num_entries_in_window (inp ,16 )) 5932 ) 61print_entry_window_info (num_entries_in_window (inp ,32 )) 6264 ) 64print_entry_window_info (num_entries_in_window (inp ,64 ))