fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #72883 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum49cae55760d

1.44.0
11.9 KiB337 linesraw
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# This script computes the new "current" toolstate for the toolstate repo (not to be
5# confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
6# It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
7# (i.e., after it passed all checks on `auto`).
8
9from __future__ import print_function
10
11import sys
12import re
13import os
14import json
15import datetime
16import collections
17import textwrap
18try:
19    import urllib2
20except ImportError:
21    import urllib.request as urllib2
22
23# List of people to ping when the status of a tool or a book changed.
24# These should be collaborators of the rust-lang/rust repository (with at least
25# read privileges on it). CI will fail otherwise.
26MAINTAINERS = {
27    'miri': {'oli-obk', 'RalfJung', 'eddyb'},
28    'clippy-driver': {
29        'Manishearth', 'llogiq', 'mcarton', 'oli-obk', 'phansch', 'flip1995',
30        'yaahc',
31    },
32    'rls': {'Xanewok'},
33    'rustfmt': {'topecongiro'},
34    'book': {'carols10cents', 'steveklabnik'},
35    'nomicon': {'frewsxcv', 'Gankra'},
36    'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'},
37    'rust-by-example': {'steveklabnik', 'marioidival'},
38    'embedded-book': {
39        'adamgreig', 'andre-richter', 'jamesmunns', 'korken89',
40        'ryankurte', 'thejpster', 'therealprof',
41    },
42    'edition-guide': {'ehuss', 'steveklabnik'},
43    'rustc-dev-guide': {'mark-i-m', 'spastorino', 'amanjeev', 'JohnTitor'},
44}
45
46REPOS = {
47    'miri': 'https://github.com/rust-lang/miri',
48    'clippy-driver': 'https://github.com/rust-lang/rust-clippy',
49    'rls': 'https://github.com/rust-lang/rls',
50    'rustfmt': 'https://github.com/rust-lang/rustfmt',
51    'book': 'https://github.com/rust-lang/book',
52    'nomicon': 'https://github.com/rust-lang/nomicon',
53    'reference': 'https://github.com/rust-lang/reference',
54    'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
55    'embedded-book': 'https://github.com/rust-embedded/book',
56    'edition-guide': 'https://github.com/rust-lang/edition-guide',
57    'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
58}
59
60def load_json_from_response(resp):
61    content = resp.read()
62    if isinstance(content, bytes):
63        content = content.decode('utf-8')
64    else:
65        print("Refusing to decode " + str(type(content)) + " to str")
66    return json.loads(content)
67
68def validate_maintainers(repo, github_token):
69    '''Ensure all maintainers are assignable on a GitHub repo'''
70    next_link_re = re.compile(r'<([^>]+)>; rel="next"')
71
72    # Load the list of assignable people in the GitHub repo
73    assignable = []
74    url = 'https://api.github.com/repos/%s/collaborators?per_page=100' % repo
75    while url is not None:
76        response = urllib2.urlopen(urllib2.Request(url, headers={
77            'Authorization': 'token ' + github_token,
78            # Properly load nested teams.
79            'Accept': 'application/vnd.github.hellcat-preview+json',
80        }))
81        assignable.extend(user['login'] for user in load_json_from_response(response))
82        # Load the next page if available
83        url = None
84        link_header = response.headers.get('Link')
85        if link_header:
86            matches = next_link_re.match(link_header)
87            if matches is not None:
88                url = matches.group(1)
89
90    errors = False
91    for tool, maintainers in MAINTAINERS.items():
92        for maintainer in maintainers:
93            if maintainer not in assignable:
94                errors = True
95                print(
96                    "error: %s maintainer @%s is not assignable in the %s repo"
97                    % (tool, maintainer, repo),
98                )
99
100    if errors:
101        print()
102        print("  To be assignable, a person needs to be explicitly listed as a")
103        print("  collaborator in the repository settings. The simple way to")
104        print("  fix this is to ask someone with 'admin' privileges on the repo")
105        print("  to add the person or whole team as a collaborator with 'read'")
106        print("  privileges. Those privileges don't grant any extra permissions")
107        print("  so it's safe to apply them.")
108        print()
109        print("The build will fail due to this.")
110        exit(1)
111
112
113def read_current_status(current_commit, path):
114    '''Reads build status of `current_commit` from content of `history/*.tsv`
115    '''
116    with open(path, 'rU') as f:
117        for line in f:
118            (commit, status) = line.split('\t', 1)
119            if commit == current_commit:
120                return json.loads(status)
121    return {}
122
123
124def gh_url():
125    return os.environ['TOOLSTATE_ISSUES_API_URL']
126
127
128def maybe_delink(message):
129    if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
130        return message.replace("@", "")
131    return message
132
133
134def issue(
135    tool,
136    status,
137    assignees,
138    relevant_pr_number,
139    relevant_pr_user,
140):
141    # Open an issue about the toolstate failure.
142    if status == 'test-fail':
143        status_description = 'has failing tests'
144    else:
145        status_description = 'no longer builds'
146    request = json.dumps({
147        'body': maybe_delink(textwrap.dedent('''\
148        Hello, this is your friendly neighborhood mergebot.
149        After merging PR {}, I observed that the tool {} {}.
150        A follow-up PR to the repository {} is needed to fix the fallout.
151
152        cc @{}, do you think you would have time to do the follow-up work?
153        If so, that would be great!
154
155        And nominating for compiler team prioritization.
156
157        ''').format(
158            relevant_pr_number, tool, status_description,
159            REPOS.get(tool), relevant_pr_user
160        )),
161        'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
162        'assignees': list(assignees),
163    })
164    print("Creating issue:\n{}".format(request))
165    response = urllib2.urlopen(urllib2.Request(
166        gh_url(),
167        request,
168        {
169            'Authorization': 'token ' + github_token,
170            'Content-Type': 'application/json',
171        }
172    ))
173    response.read()
174
175
176def update_latest(
177    current_commit,
178    relevant_pr_number,
179    relevant_pr_url,
180    relevant_pr_user,
181    pr_reviewer,
182    current_datetime
183):
184    '''Updates `_data/latest.json` to match build result of the given commit.
185    '''
186    with open('_data/latest.json', 'r+') as f:
187        latest = json.load(f, object_pairs_hook=collections.OrderedDict)
188
189        current_status = {
190            os: read_current_status(current_commit, 'history/' + os + '.tsv')
191            for os in ['windows', 'linux']
192        }
193
194        slug = 'rust-lang/rust'
195        message = textwrap.dedent('''\
196            📣 Toolstate changed by {}!
197
198            Tested on commit {}@{}.
199            Direct link to PR: <{}>
200
201        ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
202        anything_changed = False
203        for status in latest:
204            tool = status['tool']
205            changed = False
206            create_issue_for_status = None  # set to the status that caused the issue
207
208            for os, s in current_status.items():
209                old = status[os]
210                new = s.get(tool, old)
211                status[os] = new
212                maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
213                # comparing the strings, but they are ordered appropriately:
214                # "test-pass" > "test-fail" > "build-fail"
215                if new > old:
216                    # things got fixed or at least the status quo improved
217                    changed = True
218                    message += '🎉 {} on {}: {} → {} (cc {}).\n' \
219                        .format(tool, os, old, new, maintainers)
220                elif new < old:
221                    # tests or builds are failing and were not failing before
222                    changed = True
223                    title = '💔 {} on {}: {} → {}' \
224                        .format(tool, os, old, new)
225                    message += '{} (cc {}).\n' \
226                        .format(title, maintainers)
227                    # See if we need to create an issue.
228                    if tool == 'miri':
229                        # Create issue if tests used to pass before. Don't open a *second*
230                        # issue when we regress from "test-fail" to "build-fail".
231                        if old == 'test-pass':
232                            create_issue_for_status = new
233                    else:
234                        # Create issue if things no longer build.
235                        # (No issue for mere test failures to avoid spurious issues.)
236                        if new == 'build-fail':
237                            create_issue_for_status = new
238
239            if create_issue_for_status is not None:
240                try:
241                    issue(
242                        tool, create_issue_for_status, MAINTAINERS.get(tool, ''),
243                        relevant_pr_number, relevant_pr_user,
244                    )
245                except urllib2.HTTPError as e:
246                    # network errors will simply end up not creating an issue, but that's better
247                    # than failing the entire build job
248                    print("HTTPError when creating issue for status regression: {0}\n{1}"
249                          .format(e, e.read()))
250                except IOError as e:
251                    print("I/O error when creating issue for status regression: {0}".format(e))
252                except:
253                    print("Unexpected error when creating issue for status regression: {0}"
254                          .format(sys.exc_info()[0]))
255                    raise
256
257            if changed:
258                status['commit'] = current_commit
259                status['datetime'] = current_datetime
260                anything_changed = True
261
262        if not anything_changed:
263            return ''
264
265        f.seek(0)
266        f.truncate(0)
267        json.dump(latest, f, indent=4, separators=(',', ': '))
268        return message
269
270
271if __name__ == '__main__':
272    repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
273    if repo:
274        github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
275        if github_token:
276            validate_maintainers(repo, github_token)
277        else:
278            print('skipping toolstate maintainers validation since no GitHub token is present')
279        # When validating maintainers don't run the full script.
280        exit(0)
281
282    cur_commit = sys.argv[1]
283    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
284    cur_commit_msg = sys.argv[2]
285    save_message_to_path = sys.argv[3]
286    github_token = sys.argv[4]
287
288    # assume that PR authors are also owners of the repo where the branch lives
289    relevant_pr_match = re.search(
290        r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
291        cur_commit_msg,
292    )
293    if relevant_pr_match:
294        number = relevant_pr_match.group(1)
295        relevant_pr_user = relevant_pr_match.group(2)
296        relevant_pr_number = 'rust-lang/rust#' + number
297        relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
298        pr_reviewer = relevant_pr_match.group(3)
299    else:
300        number = '-1'
301        relevant_pr_user = 'ghost'
302        relevant_pr_number = '<unknown PR>'
303        relevant_pr_url = '<unknown>'
304        pr_reviewer = 'ghost'
305
306    message = update_latest(
307        cur_commit,
308        relevant_pr_number,
309        relevant_pr_url,
310        relevant_pr_user,
311        pr_reviewer,
312        cur_datetime
313    )
314    if not message:
315        print('<Nothing changed>')
316        sys.exit(0)
317
318    print(message)
319
320    if not github_token:
321        print('Dry run only, not committing anything')
322        sys.exit(0)
323
324    with open(save_message_to_path, 'w') as f:
325        f.write(message)
326
327    # Write the toolstate comment on the PR as well.
328    issue_url = gh_url() + '/{}/comments'.format(number)
329    response = urllib2.urlopen(urllib2.Request(
330        issue_url,
331        json.dumps({'body': maybe_delink(message)}),
332        {
333            'Authorization': 'token ' + github_token,
334            'Content-Type': 'application/json',
335        }
336    ))
337    response.read()