fork/fish-shell

git clone https://git.julia.blue/fork/fish-shell

Fabian BoehmUnconditionally default emoji width to 2856100851

master
12.3 KiB419 linesraw
1#!/usr/bin/env python3
2import argparse
3import asyncio
4import multiprocessing
5import os
6import resource
7import shutil
8import subprocess
9import sys
10import tempfile
11from datetime import datetime
12from pathlib import Path
13
14# TODO(python>3.8): use dict
15# TODO(python>3.8): use |
16from typing import Dict, Optional, Union
17
18import littlecheck
19
20try:
21    import pexpect
22
23    PEXPECT = True
24except ImportError:
25    PEXPECT = False
26
27RESET = "\033[0m"
28GREEN = "\033[32m"
29YELLOW = "\033[33m"
30BLUE = "\033[34m"
31RED = "\033[31m"
32
33IS_CYGWIN = os.uname().sysname.startswith(("CYGWIN_", "MSYS_"))
34
35
36def makeenv(script_path: Path, home: Path) -> Dict[str, str]:
37    xdg_config = home / "xdg_config_home"
38    func_dir = xdg_config / "fish" / "functions"
39    os.makedirs(func_dir)
40    os.makedirs(xdg_config / "fish" / "conf.d")
41    for func in (script_path / "test_functions").glob("*.fish"):
42        shutil.copy(func, func_dir / func.parts[-1])
43    shutil.copy(
44        script_path / "interactive.config",
45        xdg_config / "fish" / "conf.d" / "interactive.fish",
46    )
47
48    xdg_data = home / "xdg_data_home"
49    os.makedirs(xdg_data)
50    xdg_runtime = home / "xdg_runtime_home"
51    os.makedirs(xdg_runtime)
52    xdg_cache = home / "xdg_cache_home"
53    os.makedirs(xdg_cache)
54    tmp = home / "temp"
55    os.makedirs(tmp)
56
57    # Set up the environment variables for the test.
58    env = os.environ.copy()
59
60    # unset LANG, TERM, ...
61    for var in [
62        "XDG_DATA_DIRS",
63        "LANGUAGE",
64        "MC_SID",
65        "COLORTERM",
66        "KONSOLE_VERSION",
67        "STY",
68        "TERM",  # Erase this since we still respect TERM=dumb etc.
69        "TERM_PROGRAM",
70    ]:
71        if var in env:
72            del env[var]
73    langvars = [key for key in os.environ.keys() if key.startswith("LC_")]
74    for key in langvars:
75        del env[key]
76
77    env.update(
78        {
79            "HOME": str(home),
80            "TMPDIR": str(tmp),
81            "FISH_FAST_FAIL": "1",
82            "FISH_UNIT_TESTS_RUNNING": "1",
83            "XDG_CONFIG_HOME": str(xdg_config),
84            "XDG_DATA_HOME": str(xdg_data),
85            "XDG_RUNTIME_DIR": str(xdg_runtime),
86            "XDG_CACHE_HOME": str(xdg_cache),
87            "fish_test_helper": str(home.parent / "fish_test_helper"),
88            "LANG": "C",
89        }
90    )
91
92    return env
93
94
95def compile_test_helper(source_path: Path, binary_path: Path) -> None:
96    subprocess.run(
97        [
98            "cc",
99            source_path,
100            "-o",
101            binary_path,
102        ],
103        check=True,
104    )
105
106
107def is_tmux_test(path) -> bool:
108    return os.path.basename(path).startswith("tmux-")
109
110
111async def main():
112    if len(sys.argv) < 2:
113        print("Usage: test_driver.py FISH_DIRECTORY TESTS")
114        return 1
115
116    # TODO(python>3.8): no need for abspath
117    script_path = Path(os.path.abspath(__file__)).parent
118
119    argparser = argparse.ArgumentParser(
120        description="test_driver: Run fish's test suite"
121    )
122    argparser.add_argument(
123        "--max-concurrency",
124        type=int,
125        help="Maximum number of tests to run concurrently. The default is the number of logical CPUs.",
126        default=os.environ.get(
127            "FISH_TEST_MAX_CONCURRENCY", multiprocessing.cpu_count()
128        ),
129    )
130    argparser.add_argument(
131        "fish",
132        nargs=1,
133        help="Directory containing fish binaries to test (typically 'target/debug')",
134    )
135    argparser.add_argument("file", nargs="*", help="Tests to run")
136    args = argparser.parse_args()
137
138    max_concurrency = args.max_concurrency
139    if max_concurrency < 1:
140        print("--max-concurrency must be at least 1")
141        sys.exit(1)
142
143    fishdir = Path(args.fish[0]).absolute()
144    if not fishdir.is_dir():
145        fishdir = fishdir.parent
146
147    failcount = 0
148    failed = []
149    passcount = 0
150    skipcount = 0
151    def_subs = {"%": "%"}
152    lconfig = littlecheck.Config()
153    lconfig.colorize = sys.stdout.isatty()
154    lconfig.progress = True
155
156    for bin in ["fish", "fish_indent", "fish_key_reader"]:
157        if os.path.exists(fishdir / bin):
158            def_subs[bin] = str(fishdir / bin)
159        else:
160            print(f"Binary does not exist: {fishdir / bin}")
161            return 127
162
163    if args.file:
164        files = [(os.path.abspath(path), path) for path in args.file]
165    else:
166        files = [
167            (os.path.abspath(path), str(path.relative_to(Path.cwd())))
168            for path in sorted(script_path.glob("checks/*.fish"))
169            + sorted(script_path.glob("pexpects/*.py"))
170        ]
171    if os.environ.get("FISH_CI_SAN"):
172
173        def run_in_ci_san(path) -> bool:
174            if path.endswith(".py"):
175                return False
176            if is_tmux_test(path):
177                return False
178            return True
179
180        files = [path_pair for path_pair in files if run_in_ci_san(path_pair[0])]
181
182    if not PEXPECT and any(x.endswith(".py") for (x, _) in files):
183        print(f"{RED}Skipping pexpect tests because pexpect is not installed{RESET}")
184
185    if IS_CYGWIN and any(is_tmux_test(x) for (x, _) in files):
186        print(
187            f"{YELLOW}Skipping tmux tests because they are unreliable on Cygwin/MSYS{RESET}"
188        )
189
190    longest_test_name_length = max([len(arg) for _, arg in files])
191    max_expected_digits_duration = 5
192
193    def print_result(arg, result, color, duration=None, suffix=None):
194        duration_str = (
195            ""
196            if duration is None
197            else f" {str(duration_ms).rjust(max_expected_digits_duration)} ms"
198        )
199        suffix_str = "" if suffix is None else f"\n{suffix}"
200        print(
201            f"{arg.ljust(longest_test_name_length)}  {color}{result}{RESET}  {duration_str}{suffix_str}",
202            flush=True,
203        )
204
205    with tempfile.TemporaryDirectory(prefix="fishtest-root-") as tmp_root:
206        tmp_root = Path(tmp_root)
207
208        compile_test_helper(
209            script_path / "fish_test_helper.c",
210            tmp_root / "fish_test_helper",
211        )
212
213        semaphore = asyncio.Semaphore(max_concurrency)
214
215        async def run(f, arg) -> TestResult:
216            # TODO(python>3.8): use "async with"
217            if semaphore is not None:
218                await semaphore.acquire()
219            try:
220                return await run_test(
221                    tmp_root, f, arg, script_path, def_subs, lconfig, fishdir
222                )
223            finally:
224                if semaphore is not None:
225                    semaphore.release()
226
227        tasks = [create_task(run(f, arg), name=arg) for f, arg in files]
228        while tasks:
229            done, tasks = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
230            for task in done:
231                try:
232                    result = await task
233                except Exception as e:
234                    arg = task.get_name()
235                    result = TestFail(
236                        arg, None, f"Test '{arg}' raised an exception: {e}"
237                    )
238                # TODO(python>3.8): use match statement
239                if isinstance(result, TestSkip):
240                    arg = result.arg
241                    skipcount += 1
242                    print_result(arg, "SKIPPED", BLUE)
243                elif isinstance(result, TestFail):
244                    # fmt: off
245                    arg, duration_ms, error_message = result.arg, result.duration_ms, result.error_message
246                    # fmt: on
247                    failcount += 1
248                    failed += [arg]
249                    print_result(arg, "FAILED", RED, duration_ms, error_message)
250                elif isinstance(result, TestPass):
251                    arg, duration_ms = result.arg, result.duration_ms
252                    passcount += 1
253                    print_result(arg, "PASSED", GREEN, duration_ms)
254
255    if passcount + failcount + skipcount > 1:
256        print(f"{passcount} / {passcount + failcount} passed ({skipcount} skipped)")
257    if failcount:
258        failstr = "\n    ".join(failed)
259        print(f"{RED}Failed tests{RESET}:\n    {failstr}")
260    if passcount == 0 and failcount == 0 and skipcount:
261        return 125
262    return 1 if failcount else 0
263
264
265# TODO(python>=3.7): @dataclass
266class TestSkip:
267    arg: str
268
269    def __init__(self, arg: str):
270        self.arg = arg
271
272
273class TestFail:
274    arg: str
275    duration_ms: Optional[int]
276    error_message: Optional[str]
277
278    def __init__(
279        self, arg: str, duration_ms: Optional[int], error_message: Optional[str]
280    ):
281        self.arg = arg
282        self.duration_ms = duration_ms
283        self.error_message = error_message
284
285
286class TestPass:
287    arg: str
288    duration_ms: int
289
290    def __init__(self, arg: str, duration_ms: int):
291        self.arg = arg
292        self.duration_ms = duration_ms
293
294
295TestResult = Union[TestSkip, TestFail, TestPass]
296
297
298# TODO(python>3.8): use asyncio.create_task
299def create_task(coro, name: str) -> asyncio.Task:
300    task = asyncio.Task(coro)
301    if sys.version_info >= (3, 8):
302        task.set_name(name)
303    return task
304
305
306async def run_test(
307    tmp_root: Path,
308    test_file_path: str,
309    arg,
310    script_path: Path,
311    def_subs,
312    lconfig,
313    fishdir: Path,
314) -> TestResult:
315    if not test_file_path.endswith(".fish") and not test_file_path.endswith(".py"):
316        return TestFail(arg, None, f"Not a valid test file: {arg}")
317
318    starttime = datetime.now()
319    home = Path(tempfile.mkdtemp(prefix="fishtest-", dir=tmp_root))
320    test_env = makeenv(script_path, home)
321    if IS_CYGWIN and is_tmux_test(test_file_path):
322        return TestSkip(arg)
323    elif test_file_path.endswith(".fish"):
324        subs = def_subs.copy()
325        subs.update(
326            {
327                "s": test_file_path,
328                "fish_test_helper": str(tmp_root / "fish_test_helper"),
329            }
330        )
331
332        # littlecheck
333        error_message = ""
334
335        def append_error_message(x):
336            nonlocal error_message
337            error_message += x.message()
338
339        ret = await littlecheck.check_path_async(
340            test_file_path,
341            subs,
342            lconfig,
343            append_error_message,
344            env=test_env,
345            cwd=home,
346        )
347        endtime = datetime.now()
348        duration_ms = round((endtime - starttime).total_seconds() * 1000)
349        if ret is littlecheck.SKIP:
350            return TestSkip(arg)
351        elif ret:
352            return TestPass(arg, duration_ms)
353        else:
354            return TestFail(arg, duration_ms, error_message)
355    elif test_file_path.endswith(".py"):
356        test_env.update(
357            {
358                "PYTHONPATH": str(script_path),
359                "fish": str(fishdir / "fish"),
360                "fish_key_reader": str(fishdir / "fish_key_reader"),
361                "fish_indent": str(fishdir / "fish_indent"),
362                "TERM": "dumb",
363                "FISH_FORCE_COLOR": "1" if sys.stdout.isatty() else "0",
364            }
365        )
366        if not PEXPECT:
367            return TestSkip(arg)
368        PIPE = asyncio.subprocess.PIPE
369        proc = await asyncio.subprocess.create_subprocess_exec(
370            "python3",
371            test_file_path,
372            stdout=PIPE,
373            stderr=PIPE,
374            env=test_env,
375            cwd=home,
376        )
377        stdout, stderr = await proc.communicate()
378        endtime = datetime.now()
379        duration_ms = round((endtime - starttime).total_seconds() * 1000)
380        returncode = proc.returncode
381        if returncode == 0:
382            return TestPass(arg, duration_ms)
383        elif returncode == 127:
384            return TestSkip(arg)
385        else:
386            error_message = ""
387            if stdout:
388                error_message += stdout.decode("utf-8")
389            if stderr:
390                error_message += stderr.decode("utf-8")
391            return TestFail(arg, duration_ms, error_message)
392    else:
393        return TestFail(arg, None, "Error in test driver. This should be unreachable.")
394
395
396if sys.version_info < (3, 7):
397
398    def asyncio_run(coro):
399        loop = asyncio.get_event_loop()
400        try:
401            return loop.run_until_complete(coro)
402        finally:
403            if not loop.is_closed():
404                loop.close()
405
406else:
407    asyncio_run = asyncio.run
408
409if __name__ == "__main__":
410    # Increase the maximum number of open files to at least 4096,
411    # as we run tests concurrently.
412    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
413    if soft < 4096:
414        resource.setrlimit(resource.RLIMIT_NOFILE, (min(4096, hard), hard))
415    try:
416        ret = asyncio_run(main())
417        sys.exit(ret)
418    except KeyboardInterrupt:
419        sys.exit(130)