fork/rust
git clone https://git.julia.blue/fork/rust
5e1a799842b
1.41.0
1#!/usr/bin/env python 2# ignore-tidy-linelength 3 4# This is a small script that we use on CI to collect CPU usage statistics of 5# our builders. By seeing graphs of CPU usage over time we hope to correlate 6# that with possible improvements to Rust's own build system, ideally diagnosing 7# that either builders are always fully using their CPU resources or they're 8# idle for long stretches of time. 9# 10# This script is relatively simple, but it's platform specific. Each platform 11# (OSX/Windows/Linux) has a different way of calculating the current state of 12# CPU at a point in time. We then compare two captured states to determine the 13# percentage of time spent in one state versus another. The state capturing is 14# all platform-specific but the loop at the bottom is the cross platform part 15# that executes everywhere. 16# 17# # Viewing statistics 18# 19# All builders will upload their CPU statistics as CSV files to our S3 buckets. 20# These URLS look like: 21# 22# https://$bucket.s3.amazonaws.com/rustc-builds/$commit/cpu-$builder.csv 23# 24# for example 25# 26# https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/68baada19cd5340f05f0db15a3e16d6671609bcc/cpu-x86_64-apple.csv 27# 28# Each CSV file has two columns. The first is the timestamp of the measurement 29# and the second column is the % of idle cpu time in that time slice. Ideally 30# the second column is always zero. 31# 32# Once you've downloaded a file there's various ways to plot it and visualize 33# it. For command line usage you use the `src/etc/cpu-usage-over-time-plot.sh` 34# script in this repository. 35 36import datetime 37import sys 38import time 39 40if sys.platform == 'linux2': 41 class State: 42 def __init__(self): 43 with open('/proc/stat', 'r') as file: 44 data = file.readline().split() 45 if data[0] != 'cpu': 46 raise Exception('did not start with "cpu"') 47 self.user = int(data[1]) 48 self.nice = int(data[2]) 49 self.system = int(data[3]) 50 self.idle = int(data[4]) 51 self.iowait = int(data[5]) 52 self.irq = int(data[6]) 53 self.softirq = int(data[7]) 54 self.steal = int(data[8]) 55 self.guest = int(data[9]) 56 self.guest_nice = int(data[10]) 57 58 def idle_since(self, prev): 59 user = self.user - prev.user 60 nice = self.nice - prev.nice 61 system = self.system - prev.system 62 idle = self.idle - prev.idle 63 iowait = self.iowait - prev.iowait 64 irq = self.irq - prev.irq 65 softirq = self.softirq - prev.softirq 66 steal = self.steal - prev.steal 67 guest = self.guest - prev.guest 68 guest_nice = self.guest_nice - prev.guest_nice 69 total = user + nice + system + idle + iowait + irq + softirq + steal + guest + guest_nice 70 return float(idle) / float(total) * 100 71 72elif sys.platform == 'win32': 73 from ctypes.wintypes import DWORD 74 from ctypes import Structure, windll, WinError, GetLastError, byref 75 76 class FILETIME(Structure): 77 _fields_ = [ 78 ("dwLowDateTime", DWORD), 79 ("dwHighDateTime", DWORD), 80 ] 81 82 class State: 83 def __init__(self): 84 idle, kernel, user = FILETIME(), FILETIME(), FILETIME() 85 86 success = windll.kernel32.GetSystemTimes( 87 byref(idle), 88 byref(kernel), 89 byref(user), 90 ) 91 92 assert success, WinError(GetLastError())[1] 93 94 self.idle = (idle.dwHighDateTime << 32) | idle.dwLowDateTime 95 self.kernel = (kernel.dwHighDateTime << 32) | kernel.dwLowDateTime 96 self.user = (user.dwHighDateTime << 32) | user.dwLowDateTime 97 98 def idle_since(self, prev): 99 idle = self.idle - prev.idle 100 user = self.user - prev.user 101 kernel = self.kernel - prev.kernel 102 return float(idle) / float(user + kernel) * 100 103 104elif sys.platform == 'darwin': 105 from ctypes import * 106 libc = cdll.LoadLibrary('/usr/lib/libc.dylib') 107 108 PROESSOR_CPU_LOAD_INFO = c_int(2) 109 CPU_STATE_USER = 0 110 CPU_STATE_SYSTEM = 1 111 CPU_STATE_IDLE = 2 112 CPU_STATE_NICE = 3 113 c_int_p = POINTER(c_int) 114 115 class State: 116 def __init__(self): 117 num_cpus_u = c_uint(0) 118 cpu_info = c_int_p() 119 cpu_info_cnt = c_int(0) 120 err = libc.host_processor_info( 121 libc.mach_host_self(), 122 PROESSOR_CPU_LOAD_INFO, 123 byref(num_cpus_u), 124 byref(cpu_info), 125 byref(cpu_info_cnt), 126 ) 127 assert err == 0 128 self.user = 0 129 self.system = 0 130 self.idle = 0 131 self.nice = 0 132 cur = 0 133 while cur < cpu_info_cnt.value: 134 self.user += cpu_info[cur + CPU_STATE_USER] 135 self.system += cpu_info[cur + CPU_STATE_SYSTEM] 136 self.idle += cpu_info[cur + CPU_STATE_IDLE] 137 self.nice += cpu_info[cur + CPU_STATE_NICE] 138 cur += num_cpus_u.value 139 140 def idle_since(self, prev): 141 user = self.user - prev.user 142 system = self.system - prev.system 143 idle = self.idle - prev.idle 144 nice = self.nice - prev.nice 145 return float(idle) / float(user + system + idle + nice) * 100.0 146 147else: 148 print('unknown platform', sys.platform) 149 sys.exit(1) 150 151cur_state = State(); 152print("Time,Idle") 153while True: 154 time.sleep(1); 155 next_state = State(); 156 now = datetime.datetime.utcnow().isoformat() 157 idle = next_state.idle_since(cur_state) 158 print("%s,%s" % (now, idle)) 159 sys.stdout.flush() 160 cur_state = next_state