fork/fish-shell
git clone https://git.julia.blue/fork/fish-shell
856100851
master
1#!/usr/bin/env python3 2import argparse 3import asyncio 4import multiprocessing 5import os 6import resource 7import shutil 8import subprocess 9import sys 10import tempfile 11from datetime import datetime 12from pathlib import Path 13 14# TODO(python>3.8): use dict 15# TODO(python>3.8): use | 16from typing import Dict ,Optional ,Union 17 18import littlecheck 19 20try : 21import pexpect 22 23PEXPECT = True 24except ImportError : 25PEXPECT = False 26 27RESET = "\033[0m" 28GREEN = "\033[32m" 29YELLOW = "\033[33m" 30BLUE = "\033[34m" 31RED = "\033[31m" 32 33IS_CYGWIN = os .uname ().sysname .startswith (("CYGWIN_" ,"MSYS_" )) 34 35 36def makeenv (script_path :Path ,home :Path )-> Dict [str ,str ]: 37xdg_config = home / "xdg_config_home" 38func_dir = xdg_config / "fish" / "functions" 39os .makedirs (func_dir ) 40os .makedirs (xdg_config / "fish" / "conf.d" ) 41for func in (script_path / "test_functions" ).glob ("*.fish" ): 42shutil .copy (func ,func_dir / func .parts [- 1 ]) 43shutil .copy ( 44script_path / "interactive.config" , 45xdg_config / "fish" / "conf.d" / "interactive.fish" , 46 ) 47 48xdg_data = home / "xdg_data_home" 49os .makedirs (xdg_data ) 50xdg_runtime = home / "xdg_runtime_home" 51os .makedirs (xdg_runtime ) 52xdg_cache = home / "xdg_cache_home" 53os .makedirs (xdg_cache ) 54tmp = home / "temp" 55os .makedirs (tmp ) 56 57# Set up the environment variables for the test. 58env = os .environ .copy () 59 60# unset LANG, TERM, ... 61for var in [ 62"XDG_DATA_DIRS" , 63"LANGUAGE" , 64"MC_SID" , 65"COLORTERM" , 66"KONSOLE_VERSION" , 67"STY" , 68"TERM" ,# Erase this since we still respect TERM=dumb etc. 69"TERM_PROGRAM" , 70 ]: 71if var in env : 72del env [var ] 73langvars = [key for key in os .environ .keys ()if key .startswith ("LC_" )] 74for key in langvars : 75del env [key ] 76 77env .update ( 78 { 79"HOME" :str (home ), 80"TMPDIR" :str (tmp ), 81"FISH_FAST_FAIL" :"1" , 82"FISH_UNIT_TESTS_RUNNING" :"1" , 83"XDG_CONFIG_HOME" :str (xdg_config ), 84"XDG_DATA_HOME" :str (xdg_data ), 85"XDG_RUNTIME_DIR" :str (xdg_runtime ), 86"XDG_CACHE_HOME" :str (xdg_cache ), 87"fish_test_helper" :str (home .parent / "fish_test_helper" ), 88"LANG" :"C" , 89 } 90 ) 91 92return env 93 94 95def compile_test_helper (source_path :Path ,binary_path :Path )-> None : 96subprocess .run ( 97 [ 98"cc" , 99source_path , 100"-o" , 101binary_path , 102 ], 103check = True , 104 ) 105 106 107def is_tmux_test (path )-> bool : 108return os .path .basename (path ).startswith ("tmux-" ) 109 110 111async def main (): 112if len (sys .argv )< 2 : 113"Usage: test_driver.py FISH_DIRECTORY TESTS" ) 114return 1 115 116# TODO(python>3.8): no need for abspath 117script_path = Path (os .path .abspath (__file__ )).parent 118 119argparser = argparse .ArgumentParser ( 120description = "test_driver: Run fish's test suite" 121 ) 122argparser .add_argument ( 123"--max-concurrency" , 124type = int , 125help = "Maximum number of tests to run concurrently. The default is the number of logical CPUs." , 126default = os .environ .get ( 127"FISH_TEST_MAX_CONCURRENCY" ,multiprocessing .cpu_count () 128 ), 129 ) 130argparser .add_argument ( 131"fish" , 132nargs = 1 , 133help = "Directory containing fish binaries to test (typically 'target/debug')" , 134 ) 135argparser .add_argument ("file" ,nargs = "*" ,help = "Tests to run" ) 136args = argparser .parse_args () 137 138max_concurrency = args .max_concurrency 139if max_concurrency < 1 : 140"--max-concurrency must be at least 1" ) 141sys .exit (1 ) 142 143fishdir = Path (args .fish [0 ]).absolute () 144if not fishdir .is_dir (): 145fishdir = fishdir .parent 146 147failcount = 0 148failed = [] 149passcount = 0 150skipcount = 0 151def_subs = {"%" :"%" } 152lconfig = littlecheck .Config () 153lconfig .colorize = sys .stdout .isatty () 154lconfig .progress = True 155 156for bin in ["fish" ,"fish_indent" ,"fish_key_reader" ]: 157if os .path .exists (fishdir / bin ): 158def_subs [bin ]= str (fishdir / bin ) 159else : 160f"Binary does not exist: { fishdir / bin } " ) 161return 127 162 163if args .file : 164files = [(os .path .abspath (path ),path )for path in args .file ] 165else : 166files = [ 167 (os .path .abspath (path ),str (path .relative_to (Path .cwd ()))) 168for path in sorted (script_path .glob ("checks/*.fish" )) 169+ sorted (script_path .glob ("pexpects/*.py" )) 170 ] 171if os .environ .get ("FISH_CI_SAN" ): 172 173def run_in_ci_san (path )-> bool : 174if path .endswith (".py" ): 175return False 176if is_tmux_test (path ): 177return False 178return True 179 180files = [path_pair for path_pair in files if run_in_ci_san (path_pair [0 ])] 181 182if not PEXPECT and any (x .endswith (".py" )for (x ,_ )in files ): 183f" { RED } Skipping pexpect tests because pexpect is not installed { RESET } " ) 184 185if IS_CYGWIN and any (is_tmux_test (x )for (x ,_ )in files ): 186f" { YELLOW } Skipping tmux tests because they are unreliable on Cygwin/MSYS { RESET } " 188 ) 189 190longest_test_name_length = max ([len (arg )for _ ,arg in files ]) 191max_expected_digits_duration = 5 192 193def print_result (arg ,result ,color ,duration = None ,suffix = None ): 194duration_str = ( 195"" 196if duration is None 197else f" { str ( duration_ms ). rjust ( max_expected_digits_duration ) } ms" 198 ) 199suffix_str = "" if suffix is None else f"\n { suffix } " 200f" { arg . ljust ( longest_test_name_length ) } { color } { result } { RESET } { duration_str } { suffix_str } " , 202flush = True , 203 ) 204 205with tempfile .TemporaryDirectory (prefix = "fishtest-root-" )as tmp_root : 206tmp_root = Path (tmp_root ) 207 208compile_test_helper ( 209script_path / "fish_test_helper.c" , 210tmp_root / "fish_test_helper" , 211 ) 212 213semaphore = asyncio .Semaphore (max_concurrency ) 214 215async def run (f ,arg )-> TestResult : 216# TODO(python>3.8): use "async with" 217if semaphore is not None : 218await semaphore .acquire () 219try : 220return await run_test ( 221tmp_root ,f ,arg ,script_path ,def_subs ,lconfig ,fishdir 222 ) 223finally : 224if semaphore is not None : 225semaphore .release () 226 227tasks = [create_task (run (f ,arg ),name = arg )for f ,arg in files ] 228while tasks : 229done ,tasks = await asyncio .wait (tasks ,return_when = asyncio .FIRST_COMPLETED ) 230for task in done : 231try : 232result = await task 233except Exception as e : 234arg = task .get_name () 235result = TestFail ( 236arg ,None ,f"Test ' { arg } ' raised an exception: { e } " 237 ) 238# TODO(python>3.8): use match statement 239if isinstance (result ,TestSkip ): 240arg = result .arg 241skipcount += 1 242print_result (arg ,"SKIPPED" ,BLUE ) 243elif isinstance (result ,TestFail ): 244# fmt: off 245arg ,duration_ms ,error_message = result .arg ,result .duration_ms ,result .error_message 246# fmt: on 247failcount += 1 248failed += [arg ] 249print_result (arg ,"FAILED" ,RED ,duration_ms ,error_message ) 250elif isinstance (result ,TestPass ): 251arg ,duration_ms = result .arg ,result .duration_ms 252passcount += 1 253print_result (arg ,"PASSED" ,GREEN ,duration_ms ) 254 255if passcount + failcount + skipcount > 1 : 256f" { passcount } / { passcount + failcount } passed ( { skipcount } skipped)" ) 257if failcount : 258failstr = "\n " .join (failed ) 259f" { RED } Failed tests { RESET } :\n { failstr } " ) 260if passcount == 0 and failcount == 0 and skipcount : 261return 125 262return 1 if failcount else 0 263 264 265# TODO(python>=3.7): @dataclass 266class TestSkip : 267arg :str 268 269def __init__ (self ,arg :str ): 270self .arg = arg 271 272 273class TestFail : 274arg :str 275duration_ms :Optional [int ] 276error_message :Optional [str ] 277 278def __init__ ( 279self ,arg :str ,duration_ms :Optional [int ],error_message :Optional [str ] 280 ): 281self .arg = arg 282self .duration_ms = duration_ms 283self .error_message = error_message 284 285 286class TestPass : 287arg :str 288duration_ms :int 289 290def __init__ (self ,arg :str ,duration_ms :int ): 291self .arg = arg 292self .duration_ms = duration_ms 293 294 295TestResult = Union [TestSkip ,TestFail ,TestPass ] 296 297 298# TODO(python>3.8): use asyncio.create_task 299def create_task (coro ,name :str )-> asyncio .Task : 300task = asyncio .Task (coro ) 301if sys .version_info >= (3 ,8 ): 302task .set_name (name ) 303return task 304 305 306async def run_test ( 307tmp_root :Path , 308test_file_path :str , 309arg , 310script_path :Path , 311def_subs , 312lconfig , 313fishdir :Path , 314)-> TestResult : 315if not test_file_path .endswith (".fish" )and not test_file_path .endswith (".py" ): 316return TestFail (arg ,None ,f"Not a valid test file: { arg } " ) 317 318starttime = datetime .now () 319home = Path (tempfile .mkdtemp (prefix = "fishtest-" ,dir = tmp_root )) 320test_env = makeenv (script_path ,home ) 321if IS_CYGWIN and is_tmux_test (test_file_path ): 322return TestSkip (arg ) 323elif test_file_path .endswith (".fish" ): 324subs = def_subs .copy () 325subs .update ( 326 { 327"s" :test_file_path , 328"fish_test_helper" :str (tmp_root / "fish_test_helper" ), 329 } 330 ) 331 332# littlecheck 333error_message = "" 334 335def append_error_message (x ): 336nonlocal error_message 337error_message += x .message () 338 339ret = await littlecheck .check_path_async ( 340test_file_path , 341subs , 342lconfig , 343append_error_message , 344env = test_env , 345cwd = home , 346 ) 347endtime = datetime .now () 348duration_ms = round ((endtime - starttime ).total_seconds ()* 1000 ) 349if ret is littlecheck .SKIP : 350return TestSkip (arg ) 351elif ret : 352return TestPass (arg ,duration_ms ) 353else : 354return TestFail (arg ,duration_ms ,error_message ) 355elif test_file_path .endswith (".py" ): 356test_env .update ( 357 { 358"PYTHONPATH" :str (script_path ), 359"fish" :str (fishdir / "fish" ), 360"fish_key_reader" :str (fishdir / "fish_key_reader" ), 361"fish_indent" :str (fishdir / "fish_indent" ), 362"TERM" :"dumb" , 363"FISH_FORCE_COLOR" :"1" if sys .stdout .isatty ()else "0" , 364 } 365 ) 366if not PEXPECT : 367return TestSkip (arg ) 368PIPE = asyncio .subprocess .PIPE 369proc = await asyncio .subprocess .create_subprocess_exec ( 370"python3" , 371test_file_path , 372stdout = PIPE , 373stderr = PIPE , 374env = test_env , 375cwd = home , 376 ) 377stdout ,stderr = await proc .communicate () 378endtime = datetime .now () 379duration_ms = round ((endtime - starttime ).total_seconds ()* 1000 ) 380returncode = proc .returncode 381if returncode == 0 : 382return TestPass (arg ,duration_ms ) 383elif returncode == 127 : 384return TestSkip (arg ) 385else : 386error_message = "" 387if stdout : 388error_message += stdout .decode ("utf-8" ) 389if stderr : 390error_message += stderr .decode ("utf-8" ) 391return TestFail (arg ,duration_ms ,error_message ) 392else : 393return TestFail (arg ,None ,"Error in test driver. This should be unreachable." ) 394 395 396if sys .version_info < (3 ,7 ): 397 398def asyncio_run (coro ): 399loop = asyncio .get_event_loop () 400try : 401return loop .run_until_complete (coro ) 402finally : 403if not loop .is_closed (): 404loop .close () 405 406else : 407asyncio_run = asyncio .run 408 409if __name__ == "__main__" : 410# Increase the maximum number of open files to at least 4096, 411# as we run tests concurrently. 412soft ,hard = resource .getrlimit (resource .RLIMIT_NOFILE ) 413if soft < 4096 : 414resource .setrlimit (resource .RLIMIT_NOFILE , (min (4096 ,hard ),hard )) 415try : 416ret = asyncio_run (main ()) 417sys .exit (ret ) 418except KeyboardInterrupt : 419sys .exit (130 )