fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #71868 - Mark-Simulacrum:stable-1.43.1, r=pietroalbini8d69840ab92

1.43.1
11.8 KiB332 linesraw
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# This script publishes the new "current" toolstate in the toolstate repo (not to be
5# confused with publishing the test results, which happens in
6# `src/ci/docker/x86_64-gnu-tools/checktools.sh`).
7# It is set as callback for `src/ci/docker/x86_64-gnu-tools/repo.sh` by the CI scripts
8# when a new commit lands on `master` (i.e., after it passed all checks on `auto`).
9
10from __future__ import print_function
11
12import sys
13import re
14import os
15import json
16import datetime
17import collections
18import textwrap
19try:
20    import urllib2
21except ImportError:
22    import urllib.request as urllib2
23
24# List of people to ping when the status of a tool or a book changed.
25# These should be collaborators of the rust-lang/rust repository (with at least
26# read privileges on it). CI will fail otherwise.
27MAINTAINERS = {
28    'miri': {'oli-obk', 'RalfJung', 'eddyb'},
29    'clippy-driver': {
30        'Manishearth', 'llogiq', 'mcarton', 'oli-obk', 'phansch', 'flip1995',
31        'yaahc',
32    },
33    'rls': {'Xanewok'},
34    'rustfmt': {'topecongiro'},
35    'book': {'carols10cents', 'steveklabnik'},
36    'nomicon': {'frewsxcv', 'Gankra'},
37    'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'},
38    'rust-by-example': {'steveklabnik', 'marioidival'},
39    'embedded-book': {
40        'adamgreig', 'andre-richter', 'jamesmunns', 'korken89',
41        'ryankurte', 'thejpster', 'therealprof',
42    },
43    'edition-guide': {'ehuss', 'steveklabnik'},
44    'rustc-dev-guide': {'mark-i-m', 'spastorino', 'amanjeev', 'JohnTitor'},
45}
46
47REPOS = {
48    'miri': 'https://github.com/rust-lang/miri',
49    'clippy-driver': 'https://github.com/rust-lang/rust-clippy',
50    'rls': 'https://github.com/rust-lang/rls',
51    'rustfmt': 'https://github.com/rust-lang/rustfmt',
52    'book': 'https://github.com/rust-lang/book',
53    'nomicon': 'https://github.com/rust-lang/nomicon',
54    'reference': 'https://github.com/rust-lang/reference',
55    'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
56    'embedded-book': 'https://github.com/rust-embedded/book',
57    'edition-guide': 'https://github.com/rust-lang/edition-guide',
58    'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
59}
60
61
62def validate_maintainers(repo, github_token):
63    '''Ensure all maintainers are assignable on a GitHub repo'''
64    next_link_re = re.compile(r'<([^>]+)>; rel="next"')
65
66    # Load the list of assignable people in the GitHub repo
67    assignable = []
68    url = 'https://api.github.com/repos/%s/collaborators?per_page=100' % repo
69    while url is not None:
70        response = urllib2.urlopen(urllib2.Request(url, headers={
71            'Authorization': 'token ' + github_token,
72            # Properly load nested teams.
73            'Accept': 'application/vnd.github.hellcat-preview+json',
74        }))
75        assignable.extend(user['login'] for user in json.load(response))
76        # Load the next page if available
77        url = None
78        link_header = response.headers.get('Link')
79        if link_header:
80            matches = next_link_re.match(link_header)
81            if matches is not None:
82                url = matches.group(1)
83
84    errors = False
85    for tool, maintainers in MAINTAINERS.items():
86        for maintainer in maintainers:
87            if maintainer not in assignable:
88                errors = True
89                print(
90                    "error: %s maintainer @%s is not assignable in the %s repo"
91                    % (tool, maintainer, repo),
92                )
93
94    if errors:
95        print()
96        print("  To be assignable, a person needs to be explicitly listed as a")
97        print("  collaborator in the repository settings. The simple way to")
98        print("  fix this is to ask someone with 'admin' privileges on the repo")
99        print("  to add the person or whole team as a collaborator with 'read'")
100        print("  privileges. Those privileges don't grant any extra permissions")
101        print("  so it's safe to apply them.")
102        print()
103        print("The build will fail due to this.")
104        exit(1)
105
106
107def read_current_status(current_commit, path):
108    '''Reads build status of `current_commit` from content of `history/*.tsv`
109    '''
110    with open(path, 'rU') as f:
111        for line in f:
112            (commit, status) = line.split('\t', 1)
113            if commit == current_commit:
114                return json.loads(status)
115    return {}
116
117
118def gh_url():
119    return os.environ['TOOLSTATE_ISSUES_API_URL']
120
121
122def maybe_delink(message):
123    if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
124        return message.replace("@", "")
125    return message
126
127
128def issue(
129    tool,
130    status,
131    assignees,
132    relevant_pr_number,
133    relevant_pr_user,
134):
135    # Open an issue about the toolstate failure.
136    if status == 'test-fail':
137        status_description = 'has failing tests'
138    else:
139        status_description = 'no longer builds'
140    request = json.dumps({
141        'body': maybe_delink(textwrap.dedent('''\
142        Hello, this is your friendly neighborhood mergebot.
143        After merging PR {}, I observed that the tool {} {}.
144        A follow-up PR to the repository {} is needed to fix the fallout.
145
146        cc @{}, do you think you would have time to do the follow-up work?
147        If so, that would be great!
148
149        And nominating for compiler team prioritization.
150
151        ''').format(
152            relevant_pr_number, tool, status_description,
153            REPOS.get(tool), relevant_pr_user
154        )),
155        'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
156        'assignees': list(assignees),
157        'labels': ['T-compiler', 'I-nominated'],
158    })
159    print("Creating issue:\n{}".format(request))
160    response = urllib2.urlopen(urllib2.Request(
161        gh_url(),
162        request,
163        {
164            'Authorization': 'token ' + github_token,
165            'Content-Type': 'application/json',
166        }
167    ))
168    response.read()
169
170
171def update_latest(
172    current_commit,
173    relevant_pr_number,
174    relevant_pr_url,
175    relevant_pr_user,
176    pr_reviewer,
177    current_datetime
178):
179    '''Updates `_data/latest.json` to match build result of the given commit.
180    '''
181    with open('_data/latest.json', 'rb+') as f:
182        latest = json.load(f, object_pairs_hook=collections.OrderedDict)
183
184        current_status = {
185            os: read_current_status(current_commit, 'history/' + os + '.tsv')
186            for os in ['windows', 'linux']
187        }
188
189        slug = 'rust-lang/rust'
190        message = textwrap.dedent('''\
191            📣 Toolstate changed by {}!
192
193            Tested on commit {}@{}.
194            Direct link to PR: <{}>
195
196        ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
197        anything_changed = False
198        for status in latest:
199            tool = status['tool']
200            changed = False
201            create_issue_for_status = None  # set to the status that caused the issue
202
203            for os, s in current_status.items():
204                old = status[os]
205                new = s.get(tool, old)
206                status[os] = new
207                maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
208                # comparing the strings, but they are ordered appropriately:
209                # "test-pass" > "test-fail" > "build-fail"
210                if new > old:
211                    # things got fixed or at least the status quo improved
212                    changed = True
213                    message += '🎉 {} on {}: {} → {} (cc {}).\n' \
214                        .format(tool, os, old, new, maintainers)
215                elif new < old:
216                    # tests or builds are failing and were not failing before
217                    changed = True
218                    title = '💔 {} on {}: {} → {}' \
219                        .format(tool, os, old, new)
220                    message += '{} (cc {}).\n' \
221                        .format(title, maintainers)
222                    # See if we need to create an issue.
223                    if tool == 'miri':
224                        # Create issue if tests used to pass before. Don't open a *second*
225                        # issue when we regress from "test-fail" to "build-fail".
226                        if old == 'test-pass':
227                            create_issue_for_status = new
228                    else:
229                        # Create issue if things no longer build.
230                        # (No issue for mere test failures to avoid spurious issues.)
231                        if new == 'build-fail':
232                            create_issue_for_status = new
233
234            if create_issue_for_status is not None:
235                try:
236                    issue(
237                        tool, create_issue_for_status, MAINTAINERS.get(tool, ''),
238                        relevant_pr_number, relevant_pr_user,
239                    )
240                except urllib2.HTTPError as e:
241                    # network errors will simply end up not creating an issue, but that's better
242                    # than failing the entire build job
243                    print("HTTPError when creating issue for status regression: {0}\n{1}"
244                          .format(e, e.read()))
245                except IOError as e:
246                    print("I/O error when creating issue for status regression: {0}".format(e))
247                except:
248                    print("Unexpected error when creating issue for status regression: {0}"
249                          .format(sys.exc_info()[0]))
250                    raise
251
252            if changed:
253                status['commit'] = current_commit
254                status['datetime'] = current_datetime
255                anything_changed = True
256
257        if not anything_changed:
258            return ''
259
260        f.seek(0)
261        f.truncate(0)
262        json.dump(latest, f, indent=4, separators=(',', ': '))
263        return message
264
265
266if __name__ == '__main__':
267    repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
268    if repo:
269        github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
270        if github_token:
271            validate_maintainers(repo, github_token)
272        else:
273            print('skipping toolstate maintainers validation since no GitHub token is present')
274        # When validating maintainers don't run the full script.
275        exit(0)
276
277    cur_commit = sys.argv[1]
278    cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
279    cur_commit_msg = sys.argv[2]
280    save_message_to_path = sys.argv[3]
281    github_token = sys.argv[4]
282
283    # assume that PR authors are also owners of the repo where the branch lives
284    relevant_pr_match = re.search(
285        r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
286        cur_commit_msg,
287    )
288    if relevant_pr_match:
289        number = relevant_pr_match.group(1)
290        relevant_pr_user = relevant_pr_match.group(2)
291        relevant_pr_number = 'rust-lang/rust#' + number
292        relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
293        pr_reviewer = relevant_pr_match.group(3)
294    else:
295        number = '-1'
296        relevant_pr_user = 'ghost'
297        relevant_pr_number = '<unknown PR>'
298        relevant_pr_url = '<unknown>'
299        pr_reviewer = 'ghost'
300
301    message = update_latest(
302        cur_commit,
303        relevant_pr_number,
304        relevant_pr_url,
305        relevant_pr_user,
306        pr_reviewer,
307        cur_datetime
308    )
309    if not message:
310        print('<Nothing changed>')
311        sys.exit(0)
312
313    print(message)
314
315    if not github_token:
316        print('Dry run only, not committing anything')
317        sys.exit(0)
318
319    with open(save_message_to_path, 'w') as f:
320        f.write(message)
321
322    # Write the toolstate comment on the PR as well.
323    issue_url = gh_url() + '/{}/comments'.format(number)
324    response = urllib2.urlopen(urllib2.Request(
325        issue_url,
326        json.dumps({'body': maybe_delink(message)}),
327        {
328            'Authorization': 'token ' + github_token,
329            'Content-Type': 'application/json',
330        }
331    ))
332    response.read()