fork/fish-shell

git clone https://git.julia.blue/fork/fish-shell

Johannes AltmanningerAllow { } for command grouping, like begin / end1bf2b43d3

master
5.5 KiB158 linesraw
1# Pygments lexer for a fish command synopsis.
2#
3# Example usage:
4# echo 'string match [OPTIONS] [STRING]' | pygmentize -f terminal256 -l doc_src/fish_synopsis.py:FishSynopsisLexer -x
5
6from docutils import nodes
7from pygments.lexer import Lexer
8from pygments.token import (
9    Generic,
10    Name,
11    Operator,
12    Punctuation,
13    Text,
14)
15import re
16from sphinx.directives.code import CodeBlock
17
18
19class FishSynopsisDirective(CodeBlock):
20    """A custom directive that describes a command's grammar."""
21
22    has_content = True
23    required_arguments = 0
24
25    def run(self):
26        if self.env.app.builder.name != "man":
27            self.arguments = ["fish-synopsis"]
28            return CodeBlock.run(self)
29        lexer = FishSynopsisLexer()
30        result = nodes.line_block()
31        for start, tok, text in lexer.get_tokens_unprocessed("\n".join(self.content)):
32            if (  # Literal text.
33                (tok in (Name.Function, Name.Constant) and not text.isupper())
34                or text.startswith("-")  # Literal option, even if it's uppercase.
35                or tok in (Operator, Punctuation)
36                or text
37                == " ]"  # Tiny hack: the closing bracket of the test(1) alias is a literal.
38            ):
39                node = nodes.strong(text=text)
40            elif (
41                tok in (Name.Constant, Name.Function) and text.isupper()
42            ):  # Placeholder parameter.
43                node = nodes.emphasis(text=text)
44            else:  # Grammar metacharacter or whitespace.
45                node = nodes.inline(text=text)
46            result.append(node)
47        return [result]
48
49
50lexer_rules = [
51    (re.compile(pattern), token)
52    for pattern, token in (
53        # Hack: treat the "[ expr ]" alias of builtin test as command token (not as grammar
54        # metacharacter).  This works because we write it without spaces in the grammar (like
55        # "[OPTIONS]").
56        (r"\. |! |\[ | \]|\{ | \}", Name.Constant),
57        # Statement separators.
58        (r"\n", Text.Whitespace),
59        (r";", Punctuation),
60        (r" +", Text.Whitespace),
61        # Operators have different highlighting than commands or parameters.
62        (r"\b(and|not|or|time)\b", Operator),
63        # Keywords that are not in command position.
64        (r"\b(if|in)\b", Name.Function),
65        # Grammar metacharacters.
66        (r"[()[\]|]", Generic.Other),
67        (r"\.\.\.", Generic.Other),
68        # Parameters.
69        (r"[\w-]+", Name.Constant),
70        (r"[=%]", Name.Constant),
71        (
72            r"[<>]",
73            Name.Constant,
74        ),  # Redirection are highlighted like parameters by default.
75    )
76]
77
78
79class FishSynopsisLexer(Lexer):
80    name = "FishSynopsisLexer"
81    aliases = ["fish-synopsis"]
82
83    is_before_command_token = None
84
85    def next_token(self, rule: str, offset: int, has_continuation_line: bool):
86        for pattern, token_kind in lexer_rules:
87            m = pattern.match(rule, pos=offset)
88            if m is None:
89                continue
90            if token_kind is Name.Constant and self.is_before_command_token:
91                token_kind = Name.Function
92
93            if has_continuation_line:
94                # Traditional case: rules with continuation lines only have a single command.
95                self.is_before_command_token = False
96            else:
97                if m.group() in ("\n", ";") or token_kind is Operator:
98                    self.is_before_command_token = True
99                elif token_kind in (Name.Constant, Name.Function):
100                    self.is_before_command_token = False
101
102            return m, token_kind, m.end()
103        return None, None, offset
104
105    def get_tokens_unprocessed(self, input_text):
106        """Return a list of (start, tok, value) tuples.
107
108        start is the index into the string
109        tok is the token type (as above)
110        value is the string contents of the token
111        """
112        """
113        A synopsis consists of multiple rules.  Each rule can have continuation lines, which
114        are expected to be indented:
115
116            cmd foo [--quux]
117                    [ARGUMENT] ...
118            cmd bar
119
120        We'll split the input into rules. This is easy for a traditional synopsis because each
121        non-indented line starts a new rule.  However, we also want to support code blocks:
122
123            switch VALUE
124               [case [GLOB ...]
125                   [COMMAND ...]]
126            end
127
128        which makes this format ambiguous. Hack around this by always adding "end" to the
129        current rule, which is enough in practice.
130        """
131        rules = []
132        rule = []
133        for line in list(input_text.splitlines()) + [""]:
134            if rule and not line.startswith(" "):
135                rules.append(rule)
136                rule = []
137                if line == "end":
138                    rules[-1].append(line)
139                    continue
140            rule.append(line)
141        result = []
142        for rule in rules:
143            offset = 0
144            self.is_before_command_token = True
145            has_continuation_line = rule[-1].startswith(" ")
146            rule = "\n".join(rule) + "\n"
147            while True:
148                match, token_kind, offset = self.next_token(
149                    rule, offset, has_continuation_line
150                )
151                if match is None:
152                    break
153                text = match.group()
154                result.append((match.start(), token_kind, text))
155            assert offset == len(rule), "cannot tokenize leftover text: '{}'".format(
156                rule[offset:]
157            )
158        return result