fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #69854 - pietroalbini:stable-next, r=Centrilb8cedc00407

1.42.0
11.9 KiB328 linesraw
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4## This script publishes the new "current" toolstate in the toolstate repo (not to be
5## confused with publishing the test results, which happens in
6## `src/ci/docker/x86_64-gnu-tools/checktools.sh`).
7## It is set as callback for `src/ci/docker/x86_64-gnu-tools/repo.sh` by the CI scripts
8## when a new commit lands on `master` (i.e., after it passed all checks on `auto`).
9
10from __future__ import print_function
11
12import sys
13import re
14import os
15import json
16import datetime
17import collections
18import textwrap
19try:
20    import urllib2
21except ImportError:
22    import urllib.request as urllib2
23
24# List of people to ping when the status of a tool or a book changed.
25# These should be collaborators of the rust-lang/rust repository (with at least
26# read privileges on it). CI will fail otherwise.
27MAINTAINERS = {
28    'miri': {'oli-obk', 'RalfJung', 'eddyb'},
29    'clippy-driver': {
30        'Manishearth', 'llogiq', 'mcarton', 'oli-obk', 'phansch', 'flip1995',
31        'yaahc',
32    },
33    'rls': {'Xanewok'},
34    'rustfmt': {'topecongiro'},
35    'book': {'carols10cents', 'steveklabnik'},
36    'nomicon': {'frewsxcv', 'Gankra'},
37    'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'},
38    'rust-by-example': {'steveklabnik', 'marioidival'},
39    'embedded-book': {
40        'adamgreig', 'andre-richter', 'jamesmunns', 'korken89',
41        'ryankurte', 'thejpster', 'therealprof',
42    },
43    'edition-guide': {'ehuss', 'Centril', 'steveklabnik'},
44    'rustc-guide': {'mark-i-m', 'spastorino', 'amanjeev', 'JohnTitor'},
45}
46
47REPOS = {
48    'miri': 'https://github.com/rust-lang/miri',
49    'clippy-driver': 'https://github.com/rust-lang/rust-clippy',
50    'rls': 'https://github.com/rust-lang/rls',
51    'rustfmt': 'https://github.com/rust-lang/rustfmt',
52    'book': 'https://github.com/rust-lang/book',
53    'nomicon': 'https://github.com/rust-lang/nomicon',
54    'reference': 'https://github.com/rust-lang/reference',
55    'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
56    'embedded-book': 'https://github.com/rust-embedded/book',
57    'edition-guide': 'https://github.com/rust-lang/edition-guide',
58    'rustc-guide': 'https://github.com/rust-lang/rustc-guide',
59}
60
61
62def validate_maintainers(repo, github_token):
63    '''Ensure all maintainers are assignable on a GitHub repo'''
64    next_link_re = re.compile(r'<([^>]+)>; rel="next"')
65
66    # Load the list of assignable people in the GitHub repo
67    assignable = []
68    url = 'https://api.github.com/repos/%s/collaborators?per_page=100' % repo
69    while url is not None:
70        response = urllib2.urlopen(urllib2.Request(url, headers={
71            'Authorization': 'token ' + github_token,
72            # Properly load nested teams.
73            'Accept': 'application/vnd.github.hellcat-preview+json',
74        }))
75        assignable.extend(user['login'] for user in json.load(response))
76        # Load the next page if available
77        url = None
78        link_header = response.headers.get('Link')
79        if link_header:
80            matches = next_link_re.match(link_header)
81            if matches is not None:
82                url = matches.group(1)
83
84    errors = False
85    for tool, maintainers in MAINTAINERS.items():
86        for maintainer in maintainers:
87            if maintainer not in assignable:
88                errors = True
89                print(
90                    "error: %s maintainer @%s is not assignable in the %s repo"
91                    % (tool, maintainer, repo),
92                )
93
94    if errors:
95        print()
96        print("  To be assignable, a person needs to be explicitly listed as a")
97        print("  collaborator in the repository settings. The simple way to")
98        print("  fix this is to ask someone with 'admin' privileges on the repo")
99        print("  to add the person or whole team as a collaborator with 'read'")
100        print("  privileges. Those privileges don't grant any extra permissions")
101        print("  so it's safe to apply them.")
102        print()
103        print("The build will fail due to this.")
104        exit(1)
105
106def read_current_status(current_commit, path):
107    '''Reads build status of `current_commit` from content of `history/*.tsv`
108    '''
109    with open(path, 'rU') as f:
110        for line in f:
111            (commit, status) = line.split('\t', 1)
112            if commit == current_commit:
113                return json.loads(status)
114    return {}
115
116def gh_url():
117    return os.environ['TOOLSTATE_ISSUES_API_URL']
118
119def maybe_delink(message):
120    if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
121        return message.replace("@", "")
122    return message
123
124def issue(
125    tool,
126    status,
127    assignees,
128    relevant_pr_number,
129    relevant_pr_user,
130    pr_reviewer,
131):
132    # Open an issue about the toolstate failure.
133    if status == 'test-fail':
134        status_description = 'has failing tests'
135    else:
136        status_description = 'no longer builds'
137    request = json.dumps({
138        'body': maybe_delink(textwrap.dedent('''\
139        Hello, this is your friendly neighborhood mergebot.
140        After merging PR {}, I observed that the tool {} {}.
141        A follow-up PR to the repository {} is needed to fix the fallout.
142
143        cc @{}, do you think you would have time to do the follow-up work?
144        If so, that would be great!
145
146        cc @{}, the PR reviewer, and nominating for compiler team prioritization.
147
148        ''').format(
149            relevant_pr_number, tool, status_description,
150            REPOS.get(tool), relevant_pr_user, pr_reviewer
151        )),
152        'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
153        'assignees': list(assignees),
154        'labels': ['T-compiler', 'I-nominated'],
155    })
156    print("Creating issue:\n{}".format(request))
157    response = urllib2.urlopen(urllib2.Request(
158        gh_url(),
159        request,
160        {
161            'Authorization': 'token ' + github_token,
162            'Content-Type': 'application/json',
163        }
164    ))
165    response.read()
166
167def update_latest(
168    current_commit,
169    relevant_pr_number,
170    relevant_pr_url,
171    relevant_pr_user,
172    pr_reviewer,
173    current_datetime
174):
175    '''Updates `_data/latest.json` to match build result of the given commit.
176    '''
177    with open('_data/latest.json', 'rb+') as f:
178        latest = json.load(f, object_pairs_hook=collections.OrderedDict)
179
180        current_status = {
181            os: read_current_status(current_commit, 'history/' + os + '.tsv')
182            for os in ['windows', 'linux']
183        }
184
185        slug = 'rust-lang/rust'
186        message = textwrap.dedent('''\
187            📣 Toolstate changed by {}!
188
189            Tested on commit {}@{}.
190            Direct link to PR: <{}>
191
192        ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
193        anything_changed = False
194        for status in latest:
195            tool = status['tool']
196            changed = False
197            create_issue_for_status = None # set to the status that caused the issue
198
199            for os, s in current_status.items():
200                old = status[os]
201                new = s.get(tool, old)
202                status[os] = new
203                maintainers = ' '.join('@'+name for name in MAINTAINERS[tool])
204                # comparing the strings, but they are ordered appropriately:
205                # "test-pass" > "test-fail" > "build-fail"
206                if new > old:
207                    # things got fixed or at least the status quo improved
208                    changed = True
209                    message += '🎉 {} on {}: {} → {} (cc {}, @rust-lang/infra).\n' \
210                        .format(tool, os, old, new, maintainers)
211                elif new < old:
212                    # tests or builds are failing and were not failing before
213                    changed = True
214                    title = '💔 {} on {}: {} → {}' \
215                        .format(tool, os, old, new)
216                    message += '{} (cc {}, @rust-lang/infra).\n' \
217                        .format(title, maintainers)
218                    # See if we need to create an issue.
219                    if tool == 'miri':
220                        # Create issue if tests used to pass before. Don't open a *second*
221                        # issue when we regress from "test-fail" to "build-fail".
222                        if old == 'test-pass':
223                            create_issue_for_status = new
224                    else:
225                        # Create issue if things no longer build.
226                        # (No issue for mere test failures to avoid spurious issues.)
227                        if new == 'build-fail':
228                            create_issue_for_status = new
229
230            if create_issue_for_status is not None:
231                try:
232                    issue(
233                        tool, create_issue_for_status, MAINTAINERS.get(tool, ''),
234                        relevant_pr_number, relevant_pr_user, pr_reviewer,
235                    )
236                except urllib2.HTTPError as e:
237                    # network errors will simply end up not creating an issue, but that's better
238                    # than failing the entire build job
239                    print("HTTPError when creating issue for status regression: {0}\n{1}"
240                          .format(e, e.read()))
241                except IOError as e:
242                    print("I/O error when creating issue for status regression: {0}".format(e))
243                except:
244                    print("Unexpected error when creating issue for status regression: {0}"
245                          .format(sys.exc_info()[0]))
246                    raise
247
248            if changed:
249                status['commit'] = current_commit
250                status['datetime'] = current_datetime
251                anything_changed = True
252
253        if not anything_changed:
254            return ''
255
256        f.seek(0)
257        f.truncate(0)
258        json.dump(latest, f, indent=4, separators=(',', ': '))
259        return message
260
261
262if __name__ == '__main__':
263    repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
264    if repo:
265        github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
266        if github_token:
267            validate_maintainers(repo, github_token)
268        else:
269            print('skipping toolstate maintainers validation since no GitHub token is present')
270        # When validating maintainers don't run the full script.
271        exit(0)
272
273    cur_commit = sys.argv[1]
274    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
275    cur_commit_msg = sys.argv[2]
276    save_message_to_path = sys.argv[3]
277    github_token = sys.argv[4]
278
279    # assume that PR authors are also owners of the repo where the branch lives
280    relevant_pr_match = re.search(
281        r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
282        cur_commit_msg,
283    )
284    if relevant_pr_match:
285        number = relevant_pr_match.group(1)
286        relevant_pr_user = relevant_pr_match.group(2)
287        relevant_pr_number = 'rust-lang/rust#' + number
288        relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
289        pr_reviewer = relevant_pr_match.group(3)
290    else:
291        number = '-1'
292        relevant_pr_user = 'ghost'
293        relevant_pr_number = '<unknown PR>'
294        relevant_pr_url = '<unknown>'
295        pr_reviewer = 'ghost'
296
297    message = update_latest(
298        cur_commit,
299        relevant_pr_number,
300        relevant_pr_url,
301        relevant_pr_user,
302        pr_reviewer,
303        cur_datetime
304    )
305    if not message:
306        print('<Nothing changed>')
307        sys.exit(0)
308
309    print(message)
310
311    if not github_token:
312        print('Dry run only, not committing anything')
313        sys.exit(0)
314
315    with open(save_message_to_path, 'w') as f:
316        f.write(message)
317
318    # Write the toolstate comment on the PR as well.
319    issue_url = gh_url() + '/{}/comments'.format(number)
320    response = urllib2.urlopen(urllib2.Request(
321        issue_url,
322        json.dumps({'body': maybe_delink(message)}),
323        {
324            'Authorization': 'token ' + github_token,
325            'Content-Type': 'application/json',
326        }
327    ))
328    response.read()