#!/usr/bin/env python from collections import Counter, defaultdict import sys inp = (sys.argv[1:] or ["/tmp/_1br/1br/measurements.txt"])[0] def entry_len_dist(path: str) -> Counter[int]: with open(path, 'rb') as f: out = Counter(len(line) for line in f) return out # num entries in window => (num times seen, Counter of byte len of window) type EntryWindowInfo = dict[int, tuple[int, Counter[int]]] def num_entries_in_window(path: str, window_byte_len: int) -> EntryWindowInfo: # naive; doesn't consider chunking... out: EntryWindowInfo = defaultdict(lambda: (0, Counter())) with open(path, 'rb') as f: curr_len, count = 0, 0 for line in f: # add to current window, if fits if curr_len + len(line) <= window_byte_len: curr_len += len(line) count += 1 continue # otherwise, flush current window and start a new window: count_for_window, byte_counter = out[count] byte_counter.update([curr_len]) out[count] = (count_for_window + 1, byte_counter) curr_len, count = len(line), 1 # NOTE: if `len(line)` exceeds `window_byte_len` it'll be flushed # individually on the next iteration... but we want to record this # separately so denote as zero? if curr_len > window_byte_len: count = 0 count_for_window, byte_counter = out[count] byte_counter.update([curr_len]) out[count] = (count_for_window + 1, byte_counter) curr_len = 0 count_for_window, byte_counter = out[count] byte_counter.update([curr_len]) out[count] = (count_for_window + 1, byte_counter) return out def print_entry_window_info(info: EntryWindowInfo): for num_ents, (seen, counts) in sorted(info.items()): print(f" {num_ents:2}: {seen}") # print(f" - {counts}") avg_byte_len = sum(v * c for v, c in counts.items()) / sum(c for _, c in counts.items()) print(f" - {avg_byte_len}") print(sorted(entry_len_dist(inp).items())) print() print(16) print_entry_window_info(num_entries_in_window(inp, 16)) print() print(32) print_entry_window_info(num_entries_in_window(inp, 32)) print() print(64) print_entry_window_info(num_entries_in_window(inp, 64))