fork/fish-shell
git clone https://git.julia.blue/fork/fish-shell
1bf2b43d3
master
1# Pygments lexer for a fish command synopsis. 2# 3# Example usage: 4# echo 'string match [OPTIONS] [STRING]' | pygmentize -f terminal256 -l doc_src/fish_synopsis.py:FishSynopsisLexer -x 5 6from docutils import nodes 7from pygments .lexer import Lexer 8from pygments .token import ( 9Generic , 10Name , 11Operator , 12Punctuation , 13Text , 14) 15import re 16from sphinx .directives .code import CodeBlock 17 18 19class FishSynopsisDirective (CodeBlock ): 20"""A custom directive that describes a command's grammar.""" 21 22has_content = True 23required_arguments = 0 24 25def run (self ): 26if self .env .app .builder .name != "man" : 27self .arguments = ["fish-synopsis" ] 28return CodeBlock .run (self ) 29lexer = FishSynopsisLexer () 30result = nodes .line_block () 31for start ,tok ,text in lexer .get_tokens_unprocessed ("\n" .join (self .content )): 32if (# Literal text. 33 (tok in (Name .Function ,Name .Constant )and not text .isupper ()) 34or text .startswith ("-" )# Literal option, even if it's uppercase. 35or tok in (Operator ,Punctuation ) 36or text 37== " ]" # Tiny hack: the closing bracket of the test(1) alias is a literal. 38 ): 39node = nodes .strong (text = text ) 40elif ( 41tok in (Name .Constant ,Name .Function )and text .isupper () 42 ):# Placeholder parameter. 43node = nodes .emphasis (text = text ) 44else :# Grammar metacharacter or whitespace. 45node = nodes .inline (text = text ) 46result .append (node ) 47return [result ] 48 49 50lexer_rules = [ 51 (re .compile (pattern ),token ) 52for pattern ,token in ( 53# Hack: treat the "[ expr ]" alias of builtin test as command token (not as grammar 54# metacharacter). This works because we write it without spaces in the grammar (like 55# "[OPTIONS]"). 56 (r"\. |! |\[ | \]|\{ | \}" ,Name .Constant ), 57# Statement separators. 58 (r"\n" ,Text .Whitespace ), 59 (r";" ,Punctuation ), 60 (r" +" ,Text .Whitespace ), 61# Operators have different highlighting than commands or parameters. 62 (r"\b(and|not|or|time)\b" ,Operator ), 63# Keywords that are not in command position. 64 (r"\b(if|in)\b" ,Name .Function ), 65# Grammar metacharacters. 66 (r"[()[\]|]" ,Generic .Other ), 67 (r"\.\.\." ,Generic .Other ), 68# Parameters. 69 (r"[\w-]+" ,Name .Constant ), 70 (r"[=%]" ,Name .Constant ), 71 ( 72r"[<>]" , 73Name .Constant , 74 ),# Redirection are highlighted like parameters by default. 75 ) 76] 77 78 79class FishSynopsisLexer (Lexer ): 80name = "FishSynopsisLexer" 81aliases = ["fish-synopsis" ] 82 83is_before_command_token = None 84 85def next_token (self ,rule :str ,offset :int ,has_continuation_line :bool ): 86for pattern ,token_kind in lexer_rules : 87m = pattern .match (rule ,pos = offset ) 88if m is None : 89continue 90if token_kind is Name .Constant and self .is_before_command_token : 91token_kind = Name .Function 92 93if has_continuation_line : 94# Traditional case: rules with continuation lines only have a single command. 95self .is_before_command_token = False 96else : 97if m .group ()in ("\n" ,";" )or token_kind is Operator : 98self .is_before_command_token = True 99elif token_kind in (Name .Constant ,Name .Function ): 100self .is_before_command_token = False 101 102return m ,token_kind ,m .end () 103return None ,None ,offset 104 105def get_tokens_unprocessed (self ,input_text ): 106"""Return a list of (start, tok, value) tuples. 107 108start is the index into the string 109tok is the token type (as above) 110value is the string contents of the token 111""" 112""" 113A synopsis consists of multiple rules. Each rule can have continuation lines, which 114are expected to be indented: 115 116cmd foo [--quux] 117[ARGUMENT] ... 118cmd bar 119 120We'll split the input into rules. This is easy for a traditional synopsis because each 121non-indented line starts a new rule. However, we also want to support code blocks: 122 123switch VALUE 124[case [GLOB ...] 125[COMMAND ...]] 126end 127 128which makes this format ambiguous. Hack around this by always adding "end" to the 129current rule, which is enough in practice. 130""" 131rules = [] 132rule = [] 133for line in list (input_text .splitlines ())+ ["" ]: 134if rule and not line .startswith (" " ): 135rules .append (rule ) 136rule = [] 137if line == "end" : 138rules [- 1 ].append (line ) 139continue 140rule .append (line ) 141result = [] 142for rule in rules : 143offset = 0 144self .is_before_command_token = True 145has_continuation_line = rule [- 1 ].startswith (" " ) 146rule = "\n" .join (rule )+ "\n" 147while True : 148match ,token_kind ,offset = self .next_token ( 149rule ,offset ,has_continuation_line 150 ) 151if match is None : 152break 153text = match .group () 154result .append ((match .start (),token_kind ,text )) 155assert offset == len (rule ),"cannot tokenize leftover text: '{}'" .format ( 156rule [offset :] 157 ) 158return result