fork/rust
git clone https://git.julia.blue/fork/rust
4560ea788cb
1.39.0
1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3 4## This script publishes the new "current" toolstate in the toolstate repo (not to be 5## confused with publishing the test results, which happens in 6## `src/ci/docker/x86_64-gnu-tools/checktools.sh`). 7## It is set as callback for `src/ci/docker/x86_64-gnu-tools/repo.sh` by the CI scripts 8## when a new commit lands on `master` (i.e., after it passed all checks on `auto`). 9 10from __future__ import print_function 11 12import sys 13import re 14import os 15import json 16import datetime 17import collections 18import textwrap 19try: 20 import urllib2 21except ImportError: 22 import urllib.request as urllib2 23 24# List of people to ping when the status of a tool or a book changed. 25# These should be collaborators of the rust-lang/rust repository (with at least 26# read privileges on it). CI will fail otherwise. 27MAINTAINERS = { 28 'miri': {'oli-obk', 'RalfJung', 'eddyb'}, 29 'clippy-driver': { 30 'Manishearth', 'llogiq', 'mcarton', 'oli-obk', 'phansch', 'flip1995', 31 'yaahc', 32 }, 33 'rls': {'Xanewok'}, 34 'rustfmt': {'topecongiro'}, 35 'book': {'carols10cents', 'steveklabnik'}, 36 'nomicon': {'frewsxcv', 'Gankra'}, 37 'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'}, 38 'rust-by-example': {'steveklabnik', 'marioidival'}, 39 'embedded-book': { 40 'adamgreig', 'andre-richter', 'jamesmunns', 'korken89', 41 'ryankurte', 'thejpster', 'therealprof', 42 }, 43 'edition-guide': {'ehuss', 'Centril', 'steveklabnik'}, 44 'rustc-guide': {'mark-i-m', 'spastorino', 'amanjeev'}, 45} 46 47REPOS = { 48 'miri': 'https://github.com/rust-lang/miri', 49 'clippy-driver': 'https://github.com/rust-lang/rust-clippy', 50 'rls': 'https://github.com/rust-lang/rls', 51 'rustfmt': 'https://github.com/rust-lang/rustfmt', 52 'book': 'https://github.com/rust-lang/book', 53 'nomicon': 'https://github.com/rust-lang-nursery/nomicon', 54 'reference': 'https://github.com/rust-lang-nursery/reference', 55 'rust-by-example': 'https://github.com/rust-lang/rust-by-example', 56 'embedded-book': 'https://github.com/rust-embedded/book', 57 'edition-guide': 'https://github.com/rust-lang-nursery/edition-guide', 58 'rustc-guide': 'https://github.com/rust-lang/rustc-guide', 59} 60 61 62def validate_maintainers(repo, github_token): 63 '''Ensure all maintainers are assignable on a GitHub repo''' 64 next_link_re = re.compile(r'<([^>]+)>; rel="next"') 65 66 # Load the list of assignable people in the GitHub repo 67 assignable = [] 68 url = 'https://api.github.com/repos/%s/collaborators?per_page=100' % repo 69 while url is not None: 70 response = urllib2.urlopen(urllib2.Request(url, headers={ 71 'Authorization': 'token ' + github_token, 72 # Properly load nested teams. 73 'Accept': 'application/vnd.github.hellcat-preview+json', 74 })) 75 assignable.extend(user['login'] for user in json.load(response)) 76 # Load the next page if available 77 url = None 78 link_header = response.headers.get('Link') 79 if link_header: 80 matches = next_link_re.match(link_header) 81 if matches is not None: 82 url = matches.group(1) 83 84 errors = False 85 for tool, maintainers in MAINTAINERS.items(): 86 for maintainer in maintainers: 87 if maintainer not in assignable: 88 errors = True 89 print( 90 "error: %s maintainer @%s is not assignable in the %s repo" 91 % (tool, maintainer, repo), 92 ) 93 94 if errors: 95 print() 96 print(" To be assignable, a person needs to be explicitly listed as a") 97 print(" collaborator in the repository settings. The simple way to") 98 print(" fix this is to ask someone with 'admin' privileges on the repo") 99 print(" to add the person or whole team as a collaborator with 'read'") 100 print(" privileges. Those privileges don't grant any extra permissions") 101 print(" so it's safe to apply them.") 102 print() 103 print("The build will fail due to this.") 104 exit(1) 105 106def read_current_status(current_commit, path): 107 '''Reads build status of `current_commit` from content of `history/*.tsv` 108 ''' 109 with open(path, 'rU') as f: 110 for line in f: 111 (commit, status) = line.split('\t', 1) 112 if commit == current_commit: 113 return json.loads(status) 114 return {} 115 116def gh_url(): 117 return os.environ['TOOLSTATE_ISSUES_API_URL'] 118 119def maybe_delink(message): 120 if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None: 121 return message.replace("@", "") 122 return message 123 124def issue( 125 tool, 126 status, 127 assignees, 128 relevant_pr_number, 129 relevant_pr_user, 130 pr_reviewer, 131): 132 # Open an issue about the toolstate failure. 133 if status == 'test-fail': 134 status_description = 'has failing tests' 135 else: 136 status_description = 'no longer builds' 137 request = json.dumps({ 138 'body': maybe_delink(textwrap.dedent('''\ 139 Hello, this is your friendly neighborhood mergebot. 140 After merging PR {}, I observed that the tool {} {}. 141 A follow-up PR to the repository {} is needed to fix the fallout. 142 143 cc @{}, do you think you would have time to do the follow-up work? 144 If so, that would be great! 145 146 cc @{}, the PR reviewer, and @rust-lang/compiler -- nominating for prioritization. 147 148 ''').format( 149 relevant_pr_number, tool, status_description, 150 REPOS.get(tool), relevant_pr_user, pr_reviewer 151 )), 152 'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number), 153 'assignees': list(assignees), 154 'labels': ['T-compiler', 'I-nominated'], 155 }) 156 print("Creating issue:\n{}".format(request)) 157 response = urllib2.urlopen(urllib2.Request( 158 gh_url(), 159 request, 160 { 161 'Authorization': 'token ' + github_token, 162 'Content-Type': 'application/json', 163 } 164 )) 165 response.read() 166 167def update_latest( 168 current_commit, 169 relevant_pr_number, 170 relevant_pr_url, 171 relevant_pr_user, 172 pr_reviewer, 173 current_datetime 174): 175 '''Updates `_data/latest.json` to match build result of the given commit. 176 ''' 177 with open('_data/latest.json', 'rb+') as f: 178 latest = json.load(f, object_pairs_hook=collections.OrderedDict) 179 180 current_status = { 181 os: read_current_status(current_commit, 'history/' + os + '.tsv') 182 for os in ['windows', 'linux'] 183 } 184 185 slug = 'rust-lang/rust' 186 message = textwrap.dedent('''\ 187 📣 Toolstate changed by {}! 188 189 Tested on commit {}@{}. 190 Direct link to PR: <{}> 191 192 ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url) 193 anything_changed = False 194 for status in latest: 195 tool = status['tool'] 196 changed = False 197 create_issue_for_status = None # set to the status that caused the issue 198 199 for os, s in current_status.items(): 200 old = status[os] 201 new = s.get(tool, old) 202 status[os] = new 203 maintainers = ' '.join('@'+name for name in MAINTAINERS[tool]) 204 # comparing the strings, but they are ordered appropriately: 205 # "test-pass" > "test-fail" > "build-fail" 206 if new > old: 207 # things got fixed or at least the status quo improved 208 changed = True 209 message += '🎉 {} on {}: {} → {} (cc {}, @rust-lang/infra).\n' \ 210 .format(tool, os, old, new, maintainers) 211 elif new < old: 212 # tests or builds are failing and were not failing before 213 changed = True 214 title = '💔 {} on {}: {} → {}' \ 215 .format(tool, os, old, new) 216 message += '{} (cc {}, @rust-lang/infra).\n' \ 217 .format(title, maintainers) 218 # See if we need to create an issue. 219 if tool == 'miri': 220 # Create issue if tests used to pass before. Don't open a *second* 221 # issue when we regress from "test-fail" to "build-fail". 222 if old == 'test-pass': 223 create_issue_for_status = new 224 else: 225 # Create issue if things no longer build. 226 # (No issue for mere test failures to avoid spurious issues.) 227 if new == 'build-fail': 228 create_issue_for_status = new 229 230 if create_issue_for_status is not None: 231 try: 232 issue( 233 tool, create_issue_for_status, MAINTAINERS.get(tool, ''), 234 relevant_pr_number, relevant_pr_user, pr_reviewer, 235 ) 236 except urllib2.HTTPError as e: 237 # network errors will simply end up not creating an issue, but that's better 238 # than failing the entire build job 239 print("HTTPError when creating issue for status regression: {0}\n{1}" 240 .format(e, e.read())) 241 except IOError as e: 242 print("I/O error when creating issue for status regression: {0}".format(e)) 243 except: 244 print("Unexpected error when creating issue for status regression: {0}" 245 .format(sys.exc_info()[0])) 246 raise 247 248 if changed: 249 status['commit'] = current_commit 250 status['datetime'] = current_datetime 251 anything_changed = True 252 253 if not anything_changed: 254 return '' 255 256 f.seek(0) 257 f.truncate(0) 258 json.dump(latest, f, indent=4, separators=(',', ': ')) 259 return message 260 261 262if __name__ == '__main__': 263 repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO') 264 if repo: 265 github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN') 266 if github_token: 267 validate_maintainers(repo, github_token) 268 else: 269 print('skipping toolstate maintainers validation since no GitHub token is present') 270 # When validating maintainers don't run the full script. 271 exit(0) 272 273 cur_commit = sys.argv[1] 274 cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') 275 cur_commit_msg = sys.argv[2] 276 save_message_to_path = sys.argv[3] 277 github_token = sys.argv[4] 278 279 # assume that PR authors are also owners of the repo where the branch lives 280 relevant_pr_match = re.search( 281 r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)', 282 cur_commit_msg, 283 ) 284 if relevant_pr_match: 285 number = relevant_pr_match.group(1) 286 relevant_pr_user = relevant_pr_match.group(2) 287 relevant_pr_number = 'rust-lang/rust#' + number 288 relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number 289 pr_reviewer = relevant_pr_match.group(3) 290 else: 291 number = '-1' 292 relevant_pr_user = 'ghost' 293 relevant_pr_number = '<unknown PR>' 294 relevant_pr_url = '<unknown>' 295 pr_reviewer = 'ghost' 296 297 message = update_latest( 298 cur_commit, 299 relevant_pr_number, 300 relevant_pr_url, 301 relevant_pr_user, 302 pr_reviewer, 303 cur_datetime 304 ) 305 if not message: 306 print('<Nothing changed>') 307 sys.exit(0) 308 309 print(message) 310 311 if not github_token: 312 print('Dry run only, not committing anything') 313 sys.exit(0) 314 315 with open(save_message_to_path, 'w') as f: 316 f.write(message) 317 318 # Write the toolstate comment on the PR as well. 319 issue_url = gh_url() + '/{}/comments'.format(number) 320 response = urllib2.urlopen(urllib2.Request( 321 issue_url, 322 json.dumps({'body': maybe_delink(message)}), 323 { 324 'Authorization': 'token ' + github_token, 325 'Content-Type': 'application/json', 326 } 327 )) 328 response.read()