fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #66089 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum4560ea788cb

1.39.0
5.6 KiB160 linesraw
1#!/usr/bin/env python
2# ignore-tidy-linelength
3
4# This is a small script that we use on CI to collect CPU usage statistics of
5# our builders. By seeing graphs of CPU usage over time we hope to correlate
6# that with possible improvements to Rust's own build system, ideally diagnosing
7# that either builders are always fully using their CPU resources or they're
8# idle for long stretches of time.
9#
10# This script is relatively simple, but it's platform specific. Each platform
11# (OSX/Windows/Linux) has a different way of calculating the current state of
12# CPU at a point in time. We then compare two captured states to determine the
13# percentage of time spent in one state versus another. The state capturing is
14# all platform-specific but the loop at the bottom is the cross platform part
15# that executes everywhere.
16#
17# # Viewing statistics
18#
19# All builders will upload their CPU statistics as CSV files to our S3 buckets.
20# These URLS look like:
21#
22#   https://$bucket.s3.amazonaws.com/rustc-builds/$commit/cpu-$builder.csv
23#
24# for example
25#
26#   https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/68baada19cd5340f05f0db15a3e16d6671609bcc/cpu-x86_64-apple.csv
27#
28# Each CSV file has two columns. The first is the timestamp of the measurement
29# and the second column is the % of idle cpu time in that time slice. Ideally
30# the second column is always zero.
31#
32# Once you've downloaded a file there's various ways to plot it and visualize
33# it. For command line usage you use the `src/etc/cpu-usage-over-time-plot.sh`
34# script in this repository.
35
36import datetime
37import sys
38import time
39
40if sys.platform == 'linux2':
41    class State:
42        def __init__(self):
43            with open('/proc/stat', 'r') as file:
44                data = file.readline().split()
45            if data[0] != 'cpu':
46                raise Exception('did not start with "cpu"')
47            self.user = int(data[1])
48            self.nice = int(data[2])
49            self.system = int(data[3])
50            self.idle = int(data[4])
51            self.iowait = int(data[5])
52            self.irq = int(data[6])
53            self.softirq = int(data[7])
54            self.steal = int(data[8])
55            self.guest = int(data[9])
56            self.guest_nice = int(data[10])
57
58        def idle_since(self, prev):
59            user = self.user - prev.user
60            nice = self.nice - prev.nice
61            system = self.system - prev.system
62            idle = self.idle - prev.idle
63            iowait = self.iowait - prev.iowait
64            irq = self.irq - prev.irq
65            softirq = self.softirq - prev.softirq
66            steal = self.steal - prev.steal
67            guest = self.guest - prev.guest
68            guest_nice = self.guest_nice - prev.guest_nice
69            total = user + nice + system + idle + iowait + irq + softirq + steal + guest + guest_nice
70            return float(idle) / float(total) * 100
71
72elif sys.platform == 'win32':
73    from ctypes.wintypes import DWORD
74    from ctypes import Structure, windll, WinError, GetLastError, byref
75
76    class FILETIME(Structure):
77        _fields_ = [
78            ("dwLowDateTime", DWORD),
79            ("dwHighDateTime", DWORD),
80        ]
81
82    class State:
83        def __init__(self):
84            idle, kernel, user = FILETIME(), FILETIME(), FILETIME()
85
86            success = windll.kernel32.GetSystemTimes(
87                byref(idle),
88                byref(kernel),
89                byref(user),
90            )
91
92            assert success, WinError(GetLastError())[1]
93
94            self.idle = (idle.dwHighDateTime << 32) | idle.dwLowDateTime
95            self.kernel = (kernel.dwHighDateTime << 32) | kernel.dwLowDateTime
96            self.user = (user.dwHighDateTime << 32) | user.dwLowDateTime
97
98        def idle_since(self, prev):
99            idle = self.idle - prev.idle
100            user = self.user - prev.user
101            kernel = self.kernel - prev.kernel
102            return float(idle) / float(user + kernel) * 100
103
104elif sys.platform == 'darwin':
105    from ctypes import *
106    libc = cdll.LoadLibrary('/usr/lib/libc.dylib')
107
108    PROESSOR_CPU_LOAD_INFO = c_int(2)
109    CPU_STATE_USER = 0
110    CPU_STATE_SYSTEM = 1
111    CPU_STATE_IDLE = 2
112    CPU_STATE_NICE = 3
113    c_int_p = POINTER(c_int)
114
115    class State:
116        def __init__(self):
117            num_cpus_u = c_uint(0)
118            cpu_info = c_int_p()
119            cpu_info_cnt = c_int(0)
120            err = libc.host_processor_info(
121                libc.mach_host_self(),
122                PROESSOR_CPU_LOAD_INFO,
123                byref(num_cpus_u),
124                byref(cpu_info),
125                byref(cpu_info_cnt),
126            )
127            assert err == 0
128            self.user = 0
129            self.system = 0
130            self.idle = 0
131            self.nice = 0
132            cur = 0
133            while cur < cpu_info_cnt.value:
134                self.user += cpu_info[cur + CPU_STATE_USER]
135                self.system += cpu_info[cur + CPU_STATE_SYSTEM]
136                self.idle += cpu_info[cur + CPU_STATE_IDLE]
137                self.nice += cpu_info[cur + CPU_STATE_NICE]
138                cur += num_cpus_u.value
139
140        def idle_since(self, prev):
141            user = self.user - prev.user
142            system = self.system - prev.system
143            idle = self.idle - prev.idle
144            nice = self.nice - prev.nice
145            return float(idle) / float(user + system + idle + nice) * 100.0
146
147else:
148    print('unknown platform', sys.platform)
149    sys.exit(1)
150
151cur_state = State();
152print("Time,Idle")
153while True:
154    time.sleep(1);
155    next_state = State();
156    now = datetime.datetime.utcnow().isoformat()
157    idle = next_state.idle_since(cur_state)
158    print("%s,%s" % (now, idle))
159    sys.stdout.flush()
160    cur_state = next_state