fork/rust
git clone https://git.julia.blue/fork/rust
49cae55760d
1.44.0
1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3 4# This script computes the new "current" toolstate for the toolstate repo (not to be 5# confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`). 6# It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master` 7# (i.e., after it passed all checks on `auto`). 8 9from __future__ import print_function 10 11import sys 12import re 13import os 14import json 15import datetime 16import collections 17import textwrap 18try: 19 import urllib2 20except ImportError: 21 import urllib.request as urllib2 22 23# List of people to ping when the status of a tool or a book changed. 24# These should be collaborators of the rust-lang/rust repository (with at least 25# read privileges on it). CI will fail otherwise. 26MAINTAINERS = { 27 'miri': {'oli-obk', 'RalfJung', 'eddyb'}, 28 'clippy-driver': { 29 'Manishearth', 'llogiq', 'mcarton', 'oli-obk', 'phansch', 'flip1995', 30 'yaahc', 31 }, 32 'rls': {'Xanewok'}, 33 'rustfmt': {'topecongiro'}, 34 'book': {'carols10cents', 'steveklabnik'}, 35 'nomicon': {'frewsxcv', 'Gankra'}, 36 'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'}, 37 'rust-by-example': {'steveklabnik', 'marioidival'}, 38 'embedded-book': { 39 'adamgreig', 'andre-richter', 'jamesmunns', 'korken89', 40 'ryankurte', 'thejpster', 'therealprof', 41 }, 42 'edition-guide': {'ehuss', 'steveklabnik'}, 43 'rustc-dev-guide': {'mark-i-m', 'spastorino', 'amanjeev', 'JohnTitor'}, 44} 45 46REPOS = { 47 'miri': 'https://github.com/rust-lang/miri', 48 'clippy-driver': 'https://github.com/rust-lang/rust-clippy', 49 'rls': 'https://github.com/rust-lang/rls', 50 'rustfmt': 'https://github.com/rust-lang/rustfmt', 51 'book': 'https://github.com/rust-lang/book', 52 'nomicon': 'https://github.com/rust-lang/nomicon', 53 'reference': 'https://github.com/rust-lang/reference', 54 'rust-by-example': 'https://github.com/rust-lang/rust-by-example', 55 'embedded-book': 'https://github.com/rust-embedded/book', 56 'edition-guide': 'https://github.com/rust-lang/edition-guide', 57 'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide', 58} 59 60def load_json_from_response(resp): 61 content = resp.read() 62 if isinstance(content, bytes): 63 content = content.decode('utf-8') 64 else: 65 print("Refusing to decode " + str(type(content)) + " to str") 66 return json.loads(content) 67 68def validate_maintainers(repo, github_token): 69 '''Ensure all maintainers are assignable on a GitHub repo''' 70 next_link_re = re.compile(r'<([^>]+)>; rel="next"') 71 72 # Load the list of assignable people in the GitHub repo 73 assignable = [] 74 url = 'https://api.github.com/repos/%s/collaborators?per_page=100' % repo 75 while url is not None: 76 response = urllib2.urlopen(urllib2.Request(url, headers={ 77 'Authorization': 'token ' + github_token, 78 # Properly load nested teams. 79 'Accept': 'application/vnd.github.hellcat-preview+json', 80 })) 81 assignable.extend(user['login'] for user in load_json_from_response(response)) 82 # Load the next page if available 83 url = None 84 link_header = response.headers.get('Link') 85 if link_header: 86 matches = next_link_re.match(link_header) 87 if matches is not None: 88 url = matches.group(1) 89 90 errors = False 91 for tool, maintainers in MAINTAINERS.items(): 92 for maintainer in maintainers: 93 if maintainer not in assignable: 94 errors = True 95 print( 96 "error: %s maintainer @%s is not assignable in the %s repo" 97 % (tool, maintainer, repo), 98 ) 99 100 if errors: 101 print() 102 print(" To be assignable, a person needs to be explicitly listed as a") 103 print(" collaborator in the repository settings. The simple way to") 104 print(" fix this is to ask someone with 'admin' privileges on the repo") 105 print(" to add the person or whole team as a collaborator with 'read'") 106 print(" privileges. Those privileges don't grant any extra permissions") 107 print(" so it's safe to apply them.") 108 print() 109 print("The build will fail due to this.") 110 exit(1) 111 112 113def read_current_status(current_commit, path): 114 '''Reads build status of `current_commit` from content of `history/*.tsv` 115 ''' 116 with open(path, 'rU') as f: 117 for line in f: 118 (commit, status) = line.split('\t', 1) 119 if commit == current_commit: 120 return json.loads(status) 121 return {} 122 123 124def gh_url(): 125 return os.environ['TOOLSTATE_ISSUES_API_URL'] 126 127 128def maybe_delink(message): 129 if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None: 130 return message.replace("@", "") 131 return message 132 133 134def issue( 135 tool, 136 status, 137 assignees, 138 relevant_pr_number, 139 relevant_pr_user, 140): 141 # Open an issue about the toolstate failure. 142 if status == 'test-fail': 143 status_description = 'has failing tests' 144 else: 145 status_description = 'no longer builds' 146 request = json.dumps({ 147 'body': maybe_delink(textwrap.dedent('''\ 148 Hello, this is your friendly neighborhood mergebot. 149 After merging PR {}, I observed that the tool {} {}. 150 A follow-up PR to the repository {} is needed to fix the fallout. 151 152 cc @{}, do you think you would have time to do the follow-up work? 153 If so, that would be great! 154 155 And nominating for compiler team prioritization. 156 157 ''').format( 158 relevant_pr_number, tool, status_description, 159 REPOS.get(tool), relevant_pr_user 160 )), 161 'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number), 162 'assignees': list(assignees), 163 }) 164 print("Creating issue:\n{}".format(request)) 165 response = urllib2.urlopen(urllib2.Request( 166 gh_url(), 167 request, 168 { 169 'Authorization': 'token ' + github_token, 170 'Content-Type': 'application/json', 171 } 172 )) 173 response.read() 174 175 176def update_latest( 177 current_commit, 178 relevant_pr_number, 179 relevant_pr_url, 180 relevant_pr_user, 181 pr_reviewer, 182 current_datetime 183): 184 '''Updates `_data/latest.json` to match build result of the given commit. 185 ''' 186 with open('_data/latest.json', 'r+') as f: 187 latest = json.load(f, object_pairs_hook=collections.OrderedDict) 188 189 current_status = { 190 os: read_current_status(current_commit, 'history/' + os + '.tsv') 191 for os in ['windows', 'linux'] 192 } 193 194 slug = 'rust-lang/rust' 195 message = textwrap.dedent('''\ 196 📣 Toolstate changed by {}! 197 198 Tested on commit {}@{}. 199 Direct link to PR: <{}> 200 201 ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url) 202 anything_changed = False 203 for status in latest: 204 tool = status['tool'] 205 changed = False 206 create_issue_for_status = None # set to the status that caused the issue 207 208 for os, s in current_status.items(): 209 old = status[os] 210 new = s.get(tool, old) 211 status[os] = new 212 maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ())) 213 # comparing the strings, but they are ordered appropriately: 214 # "test-pass" > "test-fail" > "build-fail" 215 if new > old: 216 # things got fixed or at least the status quo improved 217 changed = True 218 message += '🎉 {} on {}: {} → {} (cc {}).\n' \ 219 .format(tool, os, old, new, maintainers) 220 elif new < old: 221 # tests or builds are failing and were not failing before 222 changed = True 223 title = '💔 {} on {}: {} → {}' \ 224 .format(tool, os, old, new) 225 message += '{} (cc {}).\n' \ 226 .format(title, maintainers) 227 # See if we need to create an issue. 228 if tool == 'miri': 229 # Create issue if tests used to pass before. Don't open a *second* 230 # issue when we regress from "test-fail" to "build-fail". 231 if old == 'test-pass': 232 create_issue_for_status = new 233 else: 234 # Create issue if things no longer build. 235 # (No issue for mere test failures to avoid spurious issues.) 236 if new == 'build-fail': 237 create_issue_for_status = new 238 239 if create_issue_for_status is not None: 240 try: 241 issue( 242 tool, create_issue_for_status, MAINTAINERS.get(tool, ''), 243 relevant_pr_number, relevant_pr_user, 244 ) 245 except urllib2.HTTPError as e: 246 # network errors will simply end up not creating an issue, but that's better 247 # than failing the entire build job 248 print("HTTPError when creating issue for status regression: {0}\n{1}" 249 .format(e, e.read())) 250 except IOError as e: 251 print("I/O error when creating issue for status regression: {0}".format(e)) 252 except: 253 print("Unexpected error when creating issue for status regression: {0}" 254 .format(sys.exc_info()[0])) 255 raise 256 257 if changed: 258 status['commit'] = current_commit 259 status['datetime'] = current_datetime 260 anything_changed = True 261 262 if not anything_changed: 263 return '' 264 265 f.seek(0) 266 f.truncate(0) 267 json.dump(latest, f, indent=4, separators=(',', ': ')) 268 return message 269 270 271if __name__ == '__main__': 272 repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO') 273 if repo: 274 github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN') 275 if github_token: 276 validate_maintainers(repo, github_token) 277 else: 278 print('skipping toolstate maintainers validation since no GitHub token is present') 279 # When validating maintainers don't run the full script. 280 exit(0) 281 282 cur_commit = sys.argv[1] 283 cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') 284 cur_commit_msg = sys.argv[2] 285 save_message_to_path = sys.argv[3] 286 github_token = sys.argv[4] 287 288 # assume that PR authors are also owners of the repo where the branch lives 289 relevant_pr_match = re.search( 290 r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)', 291 cur_commit_msg, 292 ) 293 if relevant_pr_match: 294 number = relevant_pr_match.group(1) 295 relevant_pr_user = relevant_pr_match.group(2) 296 relevant_pr_number = 'rust-lang/rust#' + number 297 relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number 298 pr_reviewer = relevant_pr_match.group(3) 299 else: 300 number = '-1' 301 relevant_pr_user = 'ghost' 302 relevant_pr_number = '<unknown PR>' 303 relevant_pr_url = '<unknown>' 304 pr_reviewer = 'ghost' 305 306 message = update_latest( 307 cur_commit, 308 relevant_pr_number, 309 relevant_pr_url, 310 relevant_pr_user, 311 pr_reviewer, 312 cur_datetime 313 ) 314 if not message: 315 print('<Nothing changed>') 316 sys.exit(0) 317 318 print(message) 319 320 if not github_token: 321 print('Dry run only, not committing anything') 322 sys.exit(0) 323 324 with open(save_message_to_path, 'w') as f: 325 f.write(message) 326 327 # Write the toolstate comment on the PR as well. 328 issue_url = gh_url() + '/{}/comments'.format(number) 329 response = urllib2.urlopen(urllib2.Request( 330 issue_url, 331 json.dumps({'body': maybe_delink(message)}), 332 { 333 'Authorization': 'token ' + github_token, 334 'Content-Type': 'application/json', 335 } 336 )) 337 response.read()