fork/fish-shell

git clone https://git.julia.blue/fork/fish-shell

Johannes AltmanningerMerge changes to config paths6d0cd3d9b

master
70.7 KiB2245 linesraw
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4# Run me like this: ./create_manpage_completions.py /usr/share/man/man{1,8}/* > man_completions.fish
5
6"""
7<OWNER> = Siteshwar Vashisht
8<YEAR> = 2012
9
10Copyright (c) 2012, Siteshwar Vashisht
11All rights reserved.
12
13Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
14
15Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
16Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
17THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
18"""
19
20import argparse
21import bz2
22import errno
23import gzip
24import os
25import re
26import string
27import subprocess
28import sys
29import traceback
30from subprocess import DEVNULL
31
32lzma_available = True
33try:
34    import lzma
35except ImportError:
36    lzma_available = False
37
38
39class Deroffer:
40    g_specs_specletter = {
41        # Output composed latin1 letters
42        "-D": "\320",
43        "Sd": "\360",
44        "Tp": "\376",
45        "TP": "\336",
46        "AE": "\306",
47        "ae": "\346",
48        "OE": "OE",
49        "oe": "oe",
50        ":a": "\344",
51        ":A": "\304",
52        ":e": "\353",
53        ":E": "\313",
54        ":i": "\357",
55        ":I": "\317",
56        ":o": "\366",
57        ":O": "\326",
58        ":u": "\374",
59        ":U": "\334",
60        ":y": "\377",
61        "ss": "\337",
62        "'A": "\301",
63        "'E": "\311",
64        "'I": "\315",
65        "'O": "\323",
66        "'U": "\332",
67        "'Y": "\335",
68        "'a": "\341",
69        "'e": "\351",
70        "'i": "\355",
71        "'o": "\363",
72        "'u": "\372",
73        "'y": "\375",
74        "^A": "\302",
75        "^E": "\312",
76        "^I": "\316",
77        "^O": "\324",
78        "^U": "\333",
79        "^a": "\342",
80        "^e": "\352",
81        "^i": "\356",
82        "^o": "\364",
83        "^u": "\373",
84        "`A": "\300",
85        "`E": "\310",
86        "`I": "\314",
87        "`O": "\322",
88        "`U": "\331",
89        "`a": "\340",
90        "`e": "\350",
91        "`i": "\354",
92        "`o": "\362",
93        "`u": "\371",
94        "~A": "\303",
95        "~N": "\321",
96        "~O": "\325",
97        "~a": "\343",
98        "~n": "\361",
99        "~o": "\365",
100        ",C": "\307",
101        ",c": "\347",
102        "/l": "/l",
103        "/L": "/L",
104        "/o": "\370",
105        "/O": "\330",
106        "oA": "\305",
107        "oa": "\345",
108        # Ligatures
109        "fi": "fi",
110        "ff": "ff",
111        "fl": "fl",
112        "Fi": "ffi",
113        "Ff": "fff",
114        "Fl": "ffl",
115    }
116
117    g_specs = {
118        "mi": "-",
119        "en": "-",
120        "hy": "-",
121        "em": "--",
122        "lq": "“",
123        "rq": "”",
124        "Bq": ",,",
125        "oq": "`",
126        "cq": "'",
127        "aq": "'",
128        "dq": '"',
129        "or": "|",
130        "at": "@",
131        "sh": "#",
132        "Eu": "\244",
133        "eu": "\244",
134        "Do": "$",
135        "ct": "\242",
136        "Fo": "\253",
137        "Fc": "\273",
138        "fo": "<",
139        "fc": ">",
140        "r!": "\241",
141        "r?": "\277",
142        "Of": "\252",
143        "Om": "\272",
144        "pc": "\267",
145        "S1": "\271",
146        "S2": "\262",
147        "S3": "\263",
148        "<-": "<-",
149        "->": "->",
150        "<>": "<->",
151        "ua": "^",
152        "da": "v",
153        "lA": "<=",
154        "rA": "=>",
155        "hA": "<=>",
156        "uA": "^^",
157        "dA": "vv",
158        "ba": "|",
159        "bb": "|",
160        "br": "|",
161        "bv": "|",
162        "ru": "_",
163        "ul": "_",
164        "ci": "O",
165        "bu": "o",
166        "co": "\251",
167        "rg": "\256",
168        "tm": "(TM)",
169        "dd": "||",
170        "dg": "|",
171        "ps": "\266",
172        "sc": "\247",
173        "de": "\260",
174        "%0": "0/00",
175        "14": "\274",
176        "12": "\275",
177        "34": "\276",
178        "f/": "/",
179        "sl": "/",
180        "rs": "\\",
181        "sq": "[]",
182        "fm": "'",
183        "ha": "^",
184        "ti": "~",
185        "lB": "[",
186        "rB": "]",
187        "lC": "{",
188        "rC": "}",
189        "la": "<",
190        "ra": ">",
191        "lh": "<=",
192        "rh": "=>",
193        "tf": "therefore",
194        "~~": "~~",
195        "~=": "~=",
196        "!=": "!=",
197        "**": "*",
198        "+-": "\261",
199        "<=": "<=",
200        "==": "==",
201        "=~": "=~",
202        ">=": ">=",
203        "AN": "\\/",
204        "OR": "/\\",
205        "no": "\254",
206        "te": "there exists",
207        "fa": "for all",
208        "Ah": "aleph",
209        "Im": "imaginary",
210        "Re": "real",
211        "if": "infinity",
212        "md": "\267",
213        "mo": "member of",
214        "mu": "\327",
215        "nm": "not member of",
216        "pl": "+",
217        "eq": "=",
218        "pt": "oc",
219        "pp": "perpendicular",
220        "sb": "(=",
221        "sp": "=)",
222        "ib": "(-",
223        "ip": "-)",
224        "ap": "~",
225        "is": "I",
226        "sr": "root",
227        "pd": "d",
228        "c*": "(x)",
229        "c+": "(+)",
230        "ca": "cap",
231        "cu": "U",
232        "di": "\367",
233        "gr": "V",
234        "es": "{}",
235        "CR": "_|",
236        "st": "such that",
237        "/_": "/_",
238        "lz": "<>",
239        "an": "-",
240        # Output Greek
241        "*A": "Alpha",
242        "*B": "Beta",
243        "*C": "Xi",
244        "*D": "Delta",
245        "*E": "Epsilon",
246        "*F": "Phi",
247        "*G": "Gamma",
248        "*H": "Theta",
249        "*I": "Iota",
250        "*K": "Kappa",
251        "*L": "Lambda",
252        "*M": "Mu",
253        "*N": "Nu",
254        "*O": "Omicron",
255        "*P": "Pi",
256        "*Q": "Psi",
257        "*R": "Rho",
258        "*S": "Sigma",
259        "*T": "Tau",
260        "*U": "Upsilon",
261        "*W": "Omega",
262        "*X": "Chi",
263        "*Y": "Eta",
264        "*Z": "Zeta",
265        "*a": "alpha",
266        "*b": "beta",
267        "*c": "xi",
268        "*d": "delta",
269        "*e": "epsilon",
270        "*f": "phi",
271        "+f": "phi",
272        "*g": "gamma",
273        "*h": "theta",
274        "+h": "theta",
275        "*i": "iota",
276        "*k": "kappa",
277        "*l": "lambda",
278        "*m": "\265",
279        "*n": "nu",
280        "*o": "omicron",
281        "*p": "pi",
282        "+p": "omega",
283        "*q": "psi",
284        "*r": "rho",
285        "*s": "sigma",
286        "*t": "tau",
287        "*u": "upsilon",
288        "*w": "omega",
289        "*x": "chi",
290        "*y": "eta",
291        "*z": "zeta",
292        "ts": "sigma",
293    }
294
295    g_re_word = re.compile(r"[a-zA-Z_]+")  # equivalent to the word() method
296    g_re_number = re.compile(r"[+-]?\d+")  # equivalent to the number() method
297    g_re_esc_char = re.compile(
298        # novermin
299        r"""([a-zA-Z_]) |   # Word
300                                   ([+-]?\d)   |   # Number
301                                   \\              # Backslash (for escape seq)
302                               """,
303        re.VERBOSE,
304    )
305
306    g_re_not_backslash_or_whitespace = re.compile(
307        r"[^ \t\n\r\f\v\\]+"
308    )  # Match a sequence of not backslash or whitespace
309
310    g_re_newline_collapse = re.compile(r"\n{3,}")
311
312    g_re_font = re.compile(
313        # novermin
314        r"""\\f(         # Starts with backslash f
315                               (\(\S{2}) |  # Open paren, then two printable chars
316                               (\[\S*?\]) |  # Open bracket, zero or more printable characters, then close bracket
317                               \S)          # Any printable character
318                            """,
319        re.VERBOSE,
320    )
321
322    # This gets filled in in __init__ below
323    g_macro_dict = False
324
325    def __init__(self):
326        self.reg_table = {}
327        self.tr_from = ""
328        self.tr_to = ""
329        self.tr = ""
330        self.nls = 2
331        self.specletter = False
332        self.refer = False
333        self.macro = 0
334        self.nobody = False
335        self.inlist = False
336        self.inheader = False
337        self.pic = False
338        self.tbl = False
339        self.tblstate = 0
340        self.tblTab = ""
341        self.eqn = False
342        self.skipheaders = False
343        self.skiplists = False
344        self.ignore_sonx = False
345        self.output = []
346        self.name = ""
347
348        self.OPTIONS = 0
349        self.FORMAT = 1
350        self.DATA = 2
351
352        # words is uninteresting and should be treated as false
353
354        if not Deroffer.g_macro_dict:
355            Deroffer.g_macro_dict = {
356                "SH": Deroffer.macro_sh,
357                "SS": Deroffer.macro_ss_ip,
358                "IP": Deroffer.macro_ss_ip,
359                "H ": Deroffer.macro_ss_ip,
360                "I ": Deroffer.macro_i_ir,
361                "IR": Deroffer.macro_i_ir,
362                "IB": Deroffer.macro_i_ir,
363                "B ": Deroffer.macro_i_ir,
364                "BR": Deroffer.macro_i_ir,
365                "BI": Deroffer.macro_i_ir,
366                "R ": Deroffer.macro_i_ir,
367                "RB": Deroffer.macro_i_ir,
368                "RI": Deroffer.macro_i_ir,
369                "AB": Deroffer.macro_i_ir,
370                "Nm": Deroffer.macro_Nm,
371                "] ": Deroffer.macro_close_bracket,
372                "PS": Deroffer.macro_ps,
373                "PE": Deroffer.macro_pe,
374                "TS": Deroffer.macro_ts,
375                "T&": Deroffer.macro_t_and,
376                "TE": Deroffer.macro_te,
377                "EQ": Deroffer.macro_eq,
378                "EN": Deroffer.macro_en,
379                "R1": Deroffer.macro_r1,
380                "R2": Deroffer.macro_r2,
381                "de": Deroffer.macro_de,
382                "BL": Deroffer.macro_bl_vl,
383                "VL": Deroffer.macro_bl_vl,
384                "AL": Deroffer.macro_bl_vl,
385                "LB": Deroffer.macro_bl_vl,
386                "RL": Deroffer.macro_bl_vl,
387                "ML": Deroffer.macro_bl_vl,
388                "DL": Deroffer.macro_bl_vl,
389                "BV": Deroffer.macro_bv,
390                "LE": Deroffer.macro_le,
391                "LP": Deroffer.macro_lp_pp,
392                "PP": Deroffer.macro_lp_pp,
393                "P\n": Deroffer.macro_lp_pp,
394                "ds": Deroffer.macro_ds,
395                "so": Deroffer.macro_so_nx,
396                "nx": Deroffer.macro_so_nx,
397                "tr": Deroffer.macro_tr,
398                "sp": Deroffer.macro_sp,
399            }
400
401    def flush_output(self, where):
402        if where:
403            where.write(self.get_output())
404        self.output[:] = []
405
406    def get_output(self):
407        res = "".join(self.output)
408        clean_res = Deroffer.g_re_newline_collapse.sub("\n", res)
409        return clean_res
410
411    def putchar(self, c):
412        self.output.append(c)
413        return c
414
415    # This gets swapped in in place of condputs the first time tr gets modified
416    def condputs_tr(self, str):
417        special = (
418            self.pic
419            or self.eqn
420            or self.refer
421            or self.macro
422            or (self.skiplists and self.inlist)
423            or (self.skipheaders and self.inheader)
424        )
425        if not special:
426            self.output.append(str.translate(self.tr))
427
428    def condputs(self, str):
429        special = (
430            self.pic
431            or self.eqn
432            or self.refer
433            or self.macro
434            or (self.skiplists and self.inlist)
435            or (self.skipheaders and self.inheader)
436        )
437        if not special:
438            self.output.append(str)
439
440    def str_at(self, idx):
441        return self.s[idx : idx + 1]
442
443    def skip_char(self, amt=1):
444        self.s = self.s[amt:]
445
446    def skip_leading_whitespace(self):
447        self.s = self.s.lstrip()
448
449    def is_white(self, idx):
450        # Note this returns false for empty strings (idx >= len(self.s))
451        return self.s[idx : idx + 1].isspace()
452
453    def str_eq(self, offset, other, len):
454        return self.s[offset : offset + len] == other[:len]
455
456    def prch(self, idx):
457        # Note that this return False for the empty string (idx >= len(self.s))
458        ch = self.s[idx : idx + 1]
459        return ch not in " \t\n"
460
461    def font(self):
462        match = Deroffer.g_re_font.match(self.s)
463        if not match:
464            return False
465        self.skip_char(match.end())
466        return True
467
468    def font2(self):
469        if self.s[0:2] == "\\f":
470            c = self.str_at(2)
471            if c == "(" and self.prch(3) and self.prch(4):
472                self.skip_char(5)
473                return True
474            elif c == "[":
475                self.skip_char(2)
476                while self.prch(0) and self.str_at(0) != "]":
477                    self.skip_char()
478                if self.str_at(0) == "]":
479                    self.skip_char()
480            elif self.prch(2):
481                self.skip_char(3)
482                return True
483        return False
484
485    def comment(self):
486        # Here we require that the string start with \"
487        while self.str_at(0) and self.str_at(0) != "\n":
488            self.skip_char()
489        return True
490
491    def numreq(self):
492        # We require that the string starts with backslash
493        if self.str_at(1) in "hvwud" and self.str_at(2) == "'":
494            self.macro += 1
495            self.skip_char(3)
496            while self.str_at(0) != "'" and self.esc_char():
497                pass  # Weird
498            if self.str_at(0) == "'":
499                self.skip_char()
500            self.macro -= 1
501            return True
502        return False
503
504    def device_control(self):
505        # groff \X'...' device control escape (and \Z'...' zero-width).
506        # help2man 1.50+ uses \X'tty: link URL' for hyperlinks.
507        # We just skip the entire escape.
508        if self.str_at(1) in "XZ" and self.str_at(2) == "'":
509            self.skip_char(3)
510            while self.str_at(0) and self.str_at(0) != "'":
511                self.skip_char()
512            if self.str_at(0) == "'":
513                self.skip_char()
514            return True
515        return False
516
517    def var(self):
518        reg = ""
519        s0s1 = self.s[0:2]
520        if s0s1 == "\\n":
521            if self.s[3:5] == "dy":
522                self.skip_char(5)
523                return True
524            elif self.str_at(2) == "(" and self.prch(3) and self.prch(4):
525                self.skip_char(5)
526                return True
527            elif self.str_at(2) == "[" and self.prch(3):
528                self.skip_char(3)
529                while self.str_at(0) and self.str_at(0) != "]":
530                    self.skip_char()
531                return True
532            elif self.prch(2):
533                self.skip_char(3)
534                return True
535        elif s0s1 == "\\*":
536            if self.str_at(2) == "(" and self.prch(3) and self.prch(4):
537                reg = self.s[3:5]
538                self.skip_char(5)
539            elif self.str_at(2) == "[" and self.prch(3):
540                self.skip_char(3)
541                while self.str_at(0) and self.str_at(0) != "]":
542                    reg = reg + self.str_at(0)
543                    self.skip_char()
544                if self.s[0:1] == "]":
545                    self.skip_char()
546                else:
547                    return False
548            elif self.prch(2):
549                reg = self.str_at(2)
550                self.skip_char(3)
551            else:
552                return False
553
554            if reg in self.reg_table:
555                old_s = self.s
556                self.s = self.reg_table[reg]
557                self.text_arg()
558                return True
559        return False
560
561    def size(self):
562        # We require that the string starts with \s
563        if self.digit(2) or (self.str_at(2) in "-+" and self.digit(3)):
564            self.skip_char(3)
565            while self.digit(0):
566                self.skip_char()
567            return True
568        return False
569
570    def spec(self):
571        self.specletter = False
572        if self.s[0:2] == "\\(" and self.prch(2) and self.prch(3):
573            key = self.s[2:4]
574            if key in Deroffer.g_specs_specletter:
575                self.condputs(Deroffer.g_specs_specletter[key])
576                self.specletter = True
577            elif key in Deroffer.g_specs:
578                self.condputs(Deroffer.g_specs[key])
579            self.skip_char(4)
580            return True
581        elif self.s.startswith("\\%"):
582            self.specletter = True
583            self.skip_char(2)
584            return True
585        else:
586            return False
587
588    def esc(self):
589        # We require that the string start with backslash
590        c = self.s[1:2]
591        if not c:
592            return False
593        if c in "eE":
594            self.condputs("\\")
595        elif c in "t":
596            self.condputs("\t")
597        elif c in "0~":
598            self.condputs(" ")
599        elif c in "|^&:":
600            pass
601        else:
602            self.condputs(c)
603        self.skip_char(2)
604        return True
605
606    def word(self):
607        got_something = False
608        while True:
609            match = Deroffer.g_re_word.match(self.s)
610            if not match:
611                break
612            got_something = True
613            self.condputs(match.group(0))
614            self.skip_char(match.end(0))
615
616            # Consume all specials
617            while self.spec():
618                if not self.specletter:
619                    break
620
621        return got_something
622
623    def text(self):
624        while True:
625            idx = self.s.find("\\")
626            if idx == -1:
627                self.condputs(self.s)
628                self.s = ""
629                break
630            else:
631                self.condputs(self.s[:idx])
632                self.skip_char(idx)
633                if not self.esc_char_backslash():
634                    self.condputs(self.str_at(0))
635                    self.skip_char()
636        return True
637
638    def letter(self, idx):
639        ch = self.str_at(idx)
640        return ch.isalpha() or ch == "_"  # underscore is used in C identifiers
641
642    def digit(self, idx):
643        ch = self.str_at(idx)
644        return ch.isdigit()
645
646    def number(self):
647        match = Deroffer.g_re_number.match(self.s)
648        if not match:
649            return False
650        else:
651            self.condputs(match.group(0))
652            self.skip_char(match.end())
653            return True
654
655    def esc_char_backslash(self):
656        # Like esc_char, but we know the string starts with a backslash
657        c = self.s[1:2]
658        if c == '"':
659            return self.comment()
660        elif c == "f":
661            return self.font()
662        elif c == "s":
663            return self.size()
664        elif c in "hvwud":
665            return self.numreq()
666        elif c in "XZ":
667            return self.device_control()
668        elif c in "n*":
669            return self.var()
670        elif c == "(":
671            return self.spec()
672        else:
673            return self.esc()
674
675    def esc_char(self):
676        if self.s[0:1] == "\\":
677            return self.esc_char_backslash()
678        return self.word() or self.number()
679
680    def quoted_arg(self):
681        if self.str_at(0) == '"':
682            self.skip_char()
683            while self.s and self.str_at(0) != '"':
684                if not self.esc_char():
685                    if self.s:
686                        self.condputs(self.str_at(0))
687                        self.skip_char()
688            return True
689        else:
690            return False
691
692    def text_arg(self):
693        # PCA: The deroff.c textArg() disallowed quotes at the start of an argument
694        # I'm not sure if this was a bug or not
695        got_something = False
696        while True:
697            match = Deroffer.g_re_not_backslash_or_whitespace.match(self.s)
698            if match:
699                # Output the characters in the match
700                self.condputs(match.group(0))
701                self.skip_char(match.end(0))
702                got_something = True
703
704            # Next is either an escape, or whitespace, or the end
705            # If it's the whitespace or the end, we're done
706            if not self.s or self.is_white(0):
707                return got_something
708
709            # Try an escape
710            if not self.esc_char():
711                # Some busted escape? Just output it
712                self.condputs(self.str_at(0))
713                self.skip_char()
714                got_something = True
715
716    def text_arg2(self):
717        if not self.esc_char():
718            if self.s and not self.is_white(0):
719                self.condputs(self.str_at(0))
720                self.skip_char()
721            else:
722                return False
723        while True:
724            if not self.esc_char():
725                if self.s and not self.is_white(0):
726                    self.condputs(self.str_at(0))
727                    self.skip_char()
728                else:
729                    return True
730
731    # Macro functions
732    def macro_sh(self):
733        for header_str in [" SYNOPSIS", ' "SYNOPSIS', " ‹BERSICHT", ' "‹BERSICHT']:
734            if self.s[2:].startswith(header_str):
735                self.inheader = True
736                break
737        else:
738            # Did not find a header string
739            self.inheader = False
740            self.nobody = True
741
742    def macro_ss_ip(self):
743        self.nobody = True
744        return False
745
746    def macro_i_ir(self):
747        return False
748
749    def macro_Nm(self):
750        if self.s == "Nm\n":
751            self.condputs(self.name)
752        else:
753            self.name = self.s[3:].strip() + " "
754        return True
755
756    def macro_close_bracket(self):
757        self.refer = False
758        return False
759
760    def macro_ps(self):
761        if self.is_white(2):
762            self.pic = True
763        self.condputs("\n")
764        return True
765
766    def macro_pe(self):
767        if self.is_white(2):
768            self.pic = False
769        self.condputs("\n")
770        return True
771
772    def macro_ts(self):
773        if self.is_white(2):
774            self.tbl, self.tblstate = True, self.OPTIONS
775        self.condputs("\n")
776        return True
777
778    def macro_t_and(self):
779        if self.is_white(2):
780            self.tbl, self.tblstate = True, self.FORMAT
781        self.condputs("\n")
782        return True
783
784    def macro_te(self):
785        if self.is_white(2):
786            self.tbl = False
787        self.condputs("\n")
788        return True
789
790    def macro_eq(self):
791        if self.is_white(2):
792            self.eqn = True
793        self.condputs("\n")
794        return True
795
796    def macro_en(self):
797        if self.is_white(2):
798            self.eqn = False
799        self.condputs("\n")
800        return True
801
802    def macro_r1(self):
803        if self.is_white(2):
804            self.refer2 = True
805        self.condputs("\n")
806        return True
807
808    def macro_r2(self):
809        if self.is_white(2):
810            self.refer2 = False
811        self.condputs("\n")
812        return True
813
814    def macro_de(self):
815        macro = True
816        self.condputs("\n")
817        return True
818
819    def macro_bl_vl(self):
820        if self.is_white(2):
821            self.inlist = True
822        self.condputs("\n")
823        return True
824
825    def macro_bv(self):
826        if self.str_at(2) == "L" and self.white(self.str_at(3)):
827            self.inlist = True
828        self.condputs("\n")
829        return True
830
831    def macro_le(self):
832        if self.is_white(2):
833            self.inlist = False
834        self.condputs("\n")
835        return True
836
837    def macro_lp_pp(self):
838        self.condputs("\n")
839        return True
840
841    def macro_ds(self):
842        self.skip_char(2)
843        self.skip_leading_whitespace()
844        if self.str_at(0):
845            # Split at whitespace
846            comps = self.s.split(None, 2)
847            if len(comps) == 2:
848                name, value = comps
849                value = value.rstrip()
850                self.reg_table[name] = value
851        self.condputs("\n")
852        return True
853
854    def macro_so_nx(self):
855        # We always ignore include directives
856        # deroff.c for some reason allowed this to fall through to the 'tr' case
857        # I think that was just a bug so I won't replicate it
858        return True
859
860    def macro_tr(self):
861        self.skip_char(2)
862        self.skip_leading_whitespace()
863        while self.s and self.str_at(0) != "\n":
864            c = self.str_at(0)
865            ns = self.str_at(1)
866            self.skip_char(2)
867            if not ns or ns == "\n":
868                ns = " "
869            self.tr_from += c
870            self.tr_to += ns
871
872        # Update our table, then swap in the slower tr-savvy condputs
873        self.tr = str.maketrans(self.tr_from, self.tr_to)
874        self.condputs = self.condputs_tr
875        return True
876
877    def macro_sp(self):
878        self.condputs("\n")
879        return True
880
881    def macro_other(self):
882        self.condputs("\n")
883        return True
884
885    def request_or_macro(self):
886        # s[0] is period or open single quote
887        self.skip_char()
888        s0 = self.s[1:2]
889        if s0 == "\\":
890            if self.str_at(1) == '"':
891                self.condputs("\n")
892                return True
893            else:
894                pass
895        elif s0 == "[":
896            self.refer = True
897            self.condputs("\n")
898            return True
899        elif s0 == "]":
900            self.refer = False
901            self.skip_char()
902            return self.text()
903        elif s0 == ".":
904            self.macro = False
905            self.condputs("\n")
906            return True
907
908        self.nobody = False
909        s0s1 = self.s[0:2]
910
911        macro_func = Deroffer.g_macro_dict.get(s0s1, Deroffer.macro_other)
912        if macro_func(self):
913            return True
914
915        if self.skipheaders and self.nobody:
916            return True
917
918        self.skip_leading_whitespace()
919        while self.s and not self.is_white(0):
920            self.skip_char()
921        self.skip_leading_whitespace()
922        while True:
923            if not self.quoted_arg() and not self.text_arg():
924                if self.s:
925                    self.condputs(self.str_at(0))
926                    self.skip_char()
927                else:
928                    return True
929
930    def request_or_macro2(self):
931        self.skip_char()
932        s0 = self.s[0:1]
933        if s0 == "\\":
934            if self.str_at(1) == '"':
935                self.condputs("\n")
936                return True
937            else:
938                pass
939        elif s0 == "[":
940            self.refer = True
941            self.condputs("\n")
942            return True
943        elif s0 == "]":
944            self.refer = False
945            self.skip_char()
946            return self.text()
947        elif s0 == ".":
948            self.macro = False
949            self.condputs("\n")
950            return True
951
952        self.nobody = False
953        s0s1 = self.s[0:2]
954        if s0s1 == "SH":
955            for header_str in [" SYNOPSIS", ' "SYNOPSIS', " ‹BERSICHT", ' "‹BERSICHT']:
956                if self.s[2:].startswith(header_str):
957                    self.inheader = True
958                    break
959            else:
960                # Did not find a header string
961                self.inheader = False
962                self.nobody = True
963        elif s0s1 in ["SS", "IP", "H "]:
964            self.nobody = True
965        elif s0s1 in ["I ", "IR", "IB", "B ", "BR", "BI", "R ", "RB", "RI", "AB"]:
966            pass
967        elif s0s1 in ["] "]:
968            self.refer = False
969        elif s0s1 in ["PS"]:
970            if self.is_white(2):
971                self.pic = True
972            self.condputs("\n")
973            return True
974        elif s0s1 in ["PE"]:
975            if self.is_white(2):
976                self.pic = False
977            self.condputs("\n")
978            return True
979        elif s0s1 in ["TS"]:
980            if self.is_white(2):
981                self.tbl, self.tblstate = True, self.OPTIONS
982            self.condputs("\n")
983            return True
984        elif s0s1 in ["T&"]:
985            if self.is_white(2):
986                self.tbl, self.tblstate = True, self.FORMAT
987            self.condputs("\n")
988            return True
989        elif s0s1 in ["TE"]:
990            if self.is_white(2):
991                self.tbl = False
992            self.condputs("\n")
993            return True
994        elif s0s1 in ["EQ"]:
995            if self.is_white(2):
996                self.eqn = True
997            self.condputs("\n")
998            return True
999        elif s0s1 in ["EN"]:
1000            if self.is_white(2):
1001                self.eqn = False
1002            self.condputs("\n")
1003            return True
1004        elif s0s1 in ["R1"]:
1005            if self.is_white(2):
1006                self.refer2 = True
1007            self.condputs("\n")
1008            return True
1009        elif s0s1 in ["R2"]:
1010            if self.is_white(2):
1011                self.refer2 = False
1012            self.condputs("\n")
1013            return True
1014        elif s0s1 in ["de"]:
1015            macro = True
1016            self.condputs("\n")
1017            return True
1018        elif s0s1 in ["BL", "VL", "AL", "LB", "RL", "ML", "DL"]:
1019            if self.is_white(2):
1020                self.inlist = True
1021            self.condputs("\n")
1022            return True
1023        elif s0s1 in ["BV"]:
1024            if self.str_at(2) == "L" and self.white(self.str_at(3)):
1025                self.inlist = True
1026            self.condputs("\n")
1027            return True
1028        elif s0s1 in ["LE"]:
1029            if self.is_white(2):
1030                self.inlist = False
1031            self.condputs("\n")
1032            return True
1033        elif s0s1 in ["LP", "PP", "P\n"]:
1034            self.condputs("\n")
1035            return True
1036        elif s0s1 in ["ds"]:
1037            self.skip_char(2)
1038            self.skip_leading_whitespace()
1039            if self.str_at(0):
1040                # Split at whitespace
1041                comps = self.s.split(None, 2)
1042                if len(comps) == 2:
1043                    name, value = comps
1044                    value = value.rstrip()
1045                    self.reg_table[name] = value
1046            self.condputs("\n")
1047            return True
1048        elif s0s1 in ["so", "nx"]:
1049            # We always ignore include directives
1050            # deroff.c for some reason allowed this to fall through to the 'tr' case
1051            # I think that was just a bug so I won't replicate it
1052            return True
1053        elif s0s1 in ["tr"]:
1054            self.skip_char(2)
1055            self.skip_leading_whitespace()
1056            while self.s and self.str_at(0) != "\n":
1057                c = self.str_at(0)
1058                ns = self.str_at(1)
1059                self.skip_char(2)
1060                if not ns or ns == "\n":
1061                    ns = " "
1062                self.tr_from += c
1063                self.tr_to += ns
1064
1065            # Update our table, then swap in the slower tr-savvy condputs
1066            self.tr = str.maketrans(self.tr_from, self.tr_to)
1067            self.condputs = self.condputs_tr
1068
1069            return True
1070        elif s0s1 in ["sp"]:
1071            self.condputs("\n")
1072            return True
1073        else:
1074            self.condputs("\n")
1075            return True
1076
1077        if self.skipheaders and self.nobody:
1078            return True
1079
1080        self.skip_leading_whitespace()
1081        while self.s and not self.is_white(0):
1082            self.skip_char()
1083        self.skip_leading_whitespace()
1084        while True:
1085            if not self.quoted_arg() and not self.text_arg():
1086                if self.s:
1087                    self.condputs(self.str_at(0))
1088                    self.skip_char()
1089                else:
1090                    return True
1091
1092    def do_tbl(self):
1093        if self.tblstate == self.OPTIONS:
1094            while self.s and self.str_at(0) != ";" and self.str_at(0) != "\n":
1095                self.skip_leading_whitespace()
1096                if not self.str_at(0).isalpha():
1097                    # deroff.c has a bug where it can loop forever here...we try to work around it
1098                    self.skip_char()
1099                else:  # Parse option
1100                    option = self.s
1101                    arg = ""
1102
1103                    idx = 0
1104                    while option[idx : idx + 1].isalpha():
1105                        idx += 1
1106
1107                    if option[idx : idx + 1] == "(":
1108                        option = option[:idx]
1109                        self.s = self.s[idx + 1 :]
1110                        arg = self.s
1111                    else:
1112                        self.s = ""
1113
1114                    if arg:
1115                        idx = arg.find(")")
1116                        if idx != -1:
1117                            arg = arg[:idx]
1118                        self.s = self.s[idx + 1 :]
1119                    else:
1120                        # self.skip_char()
1121                        pass
1122
1123                    if option.lower() == "tab":
1124                        self.tblTab = arg[0:1]
1125
1126            self.tblstate = self.FORMAT
1127            self.condputs("\n")
1128
1129        elif self.tblstate == self.FORMAT:
1130            while self.s and self.str_at(0) != "." and self.str_at(0) != "\n":
1131                self.skip_leading_whitespace()
1132                if self.str_at(0):
1133                    self.skip_char()
1134
1135            if self.str_at(0) == ".":
1136                self.tblstate = self.DATA
1137            self.condputs("\n")
1138        elif self.tblstate == self.DATA:
1139            if self.tblTab:
1140                self.s = self.s.replace(self.tblTab, "\t")
1141            self.text()
1142        return True
1143
1144    def do_line(self):
1145        if self.s[0:1] in ".'":
1146            if not self.request_or_macro():
1147                return False
1148        elif self.tbl:
1149            self.do_tbl()
1150        else:
1151            self.text()
1152        return True
1153
1154    def deroff(self, str):
1155        lines = str.split("\n")
1156        for line in lines:
1157            self.s = line + "\n"
1158            if not self.do_line():
1159                break
1160            # self.putchar('\n')
1161
1162
1163# This gets set to the name of the command that we are currently executing
1164CMDNAME = ""
1165
1166# Information used to track which of our parsers were successful
1167PARSER_INFO = {}
1168
1169# built_command writes into this global variable, yuck
1170built_command_output = []
1171
1172# Diagnostic output
1173diagnostic_output = []
1174diagnostic_indent = 0
1175
1176# Three diagnostic verbosity levels
1177VERY_VERBOSE, BRIEF_VERBOSE, NOT_VERBOSE = 2, 1, 0
1178
1179# Pick some reasonable default values for settings
1180VERBOSITY, WRITE_TO_STDOUT, DEROFF_ONLY, KEEP_FILES = NOT_VERBOSE, False, False, False
1181
1182
1183def add_diagnostic(dgn, msg_verbosity=VERY_VERBOSE):
1184    # Add a diagnostic message, if msg_verbosity <= VERBOSITY
1185    if msg_verbosity <= VERBOSITY:
1186        diagnostic_output.append("   " * diagnostic_indent + dgn)
1187
1188
1189def flush_diagnostics(where):
1190    if diagnostic_output:
1191        output_str = "\n".join(diagnostic_output)
1192        print(output_str, file=where)
1193        diagnostic_output[:] = []
1194
1195
1196# Make sure we don't output the same completion multiple times, which can happen
1197# For example, xsubpp.1.gz and xsubpp5.10.1.gz
1198# This maps commands to lists of completions
1199already_output_completions = {}
1200
1201
1202def compile_and_search(regex, input):
1203    options_section_regex = re.compile(regex, re.DOTALL)
1204    options_section_matched = re.search(options_section_regex, input)
1205    return options_section_matched
1206
1207
1208def unquote_double_quotes(data):
1209    if len(data) < 2:
1210        return data
1211    if data[0] == '"' and data[len(data) - 1] == '"':
1212        data = data[1 : len(data) - 1]
1213    return data
1214
1215
1216def unquote_single_quotes(data):
1217    if len(data) < 2:
1218        return data
1219    if data[0] == "`" and data[len(data) - 1] == "'":
1220        data = data[1 : len(data) - 1]
1221    return data
1222
1223
1224# Make a string of characters that are deemed safe in fish without needing to be escaped
1225# Note that space is not included
1226g_fish_safe_chars = frozenset(string.ascii_letters + string.digits + "_+-/:=@")
1227
1228
1229def fish_escape_single_quote(str):
1230    # Escape a string if necessary so that it can be put in single quotes
1231    # If it has no non-safe chars, there's nothing to do
1232    if g_fish_safe_chars.issuperset(str):
1233        return str
1234
1235    str = str.replace("\\", "\\\\")  # Replace one backslash with two
1236    str = str.replace(
1237        "'", "\\'"
1238    )  # Replace one single quote with a backslash-single-quote
1239    return "'" + str + "'"
1240
1241
1242def output_complete_command(cmdname, args, description, output_list):
1243    comps = ["complete -c", cmdname]
1244    comps.extend(args)
1245    if description:
1246        comps.append("-d")
1247        comps.append(description)
1248    output_list.append(" ".join(comps))
1249
1250
1251def built_command(options, description):
1252    #    print "Options are: ", options
1253    man_optionlist = re.split(' |,|"|=|[|]', options)
1254    fish_options = []
1255    for optionstr in man_optionlist:
1256        option = re.sub(r"(\[.*\])", "", optionstr)
1257        option = option.strip(" \t\r\n[](){}.,:!")
1258
1259        # Skip some problematic cases
1260        if option in ["-", "--"]:
1261            continue
1262        if any(c in "{}()" for c in option):
1263            continue
1264
1265        if option.startswith("--"):
1266            # New style long option (--recursive)
1267            fish_options.append("-l " + fish_escape_single_quote(option[2:]))
1268        elif option.startswith("-") and len(option) == 2:
1269            # New style short option (-r)
1270            fish_options.append("-s " + fish_escape_single_quote(option[1:]))
1271        elif option.startswith("-") and len(option) > 2:
1272            # Old style long option (-recursive)
1273            fish_options.append("-o " + fish_escape_single_quote(option[1:]))
1274
1275    # Determine which options are new (not already in existing_options)
1276    # Then add those to the existing options
1277    existing_options = already_output_completions.setdefault(CMDNAME, set())
1278    fish_options = [opt for opt in fish_options if opt not in existing_options]
1279    existing_options.update(fish_options)
1280
1281    # Maybe it's all for naught
1282    if not fish_options:
1283        return
1284
1285    # Here's what we'll use to truncate if necessary
1286    max_description_width = 78
1287    truncation_suffix = "…"
1288
1289    # Try to include as many whole sentences as will fit
1290    # Clean up some probably bogus escapes in the process
1291    clean_desc = description.replace("\\'", "'").replace("\\.", ".")
1292    sentences = clean_desc.split(".")
1293
1294    # Clean up "sentences" that are just whitespace
1295    # But don't let it be empty
1296    sentences = [x for x in sentences if x.strip()]
1297    if not sentences:
1298        sentences = [""]
1299
1300    udot = "."
1301    uspace = " "
1302
1303    truncated_description = sentences[0] + udot
1304    for line in sentences[1:]:
1305        if not line:
1306            continue
1307        proposed_description = truncated_description + uspace + line + udot
1308        if len(proposed_description) <= max_description_width:
1309            # It fits
1310            truncated_description = proposed_description
1311        else:
1312            # No fit
1313            break
1314    # Strip trailing dots
1315    truncated_description = truncated_description.strip(udot)
1316
1317    # If the first sentence does not fit, truncate if necessary
1318    if len(truncated_description) > max_description_width:
1319        prefix_len = max_description_width - len(truncation_suffix)
1320        truncated_description = truncated_description[:prefix_len] + truncation_suffix
1321
1322    # Escape some more things
1323    truncated_description = fish_escape_single_quote(truncated_description)
1324    escaped_cmd = fish_escape_single_quote(CMDNAME)
1325
1326    output_complete_command(
1327        escaped_cmd, fish_options, truncated_description, built_command_output
1328    )
1329
1330
1331def remove_groff_formatting(data):
1332    # Strip groff \X'...' device control escapes (help2man 1.50+ hyperlinks)
1333    # and \Z'...' zero-width escapes.
1334    data = re.sub(r"\\[XZ]'[^']*'", "", data)
1335    data = data.replace("\\fI", "")
1336    data = data.replace("\\fP", "")
1337    data = data.replace("\\f1", "")
1338    data = data.replace("\\fB", "")
1339    data = data.replace("\\fR", "")
1340    data = data.replace("\\e", "")
1341    data = re.sub(r".PD( \d+)", "", data)
1342    data = data.replace(".BI", "")
1343    data = data.replace(".BR", "")
1344    data = data.replace("0.5i", "")
1345    data = data.replace(".rb", "")
1346    data = data.replace("\\^", "")
1347    data = data.replace("{ ", "")
1348    data = data.replace(" }", "")
1349    data = data.replace(r"\ ", "")
1350    data = data.replace(r"\-", "-")
1351    data = data.replace(r"\&", "")
1352    data = data.replace(".B", "")
1353    data = data.replace(r"\-", "-")
1354    data = data.replace(".I", "")
1355    data = data.replace("\f", "")
1356    data = data.replace(r"\(oq", "'")
1357    data = data.replace(r"\(cq", "'")
1358    data = data.replace(r"\(aq", "'")
1359    data = data.replace(r"\(dq", '"')
1360    data = data.replace(r"\(lq", '"')
1361    data = data.replace(r"\(rq", '"')
1362    return data
1363
1364
1365class ManParser(object):
1366    def is_my_type(self, manpage):
1367        return False
1368
1369    def parse_man_page(self, manpage):
1370        return False
1371
1372
1373class Type1ManParser(ManParser):
1374    def is_my_type(self, manpage):
1375        return compile_and_search(r'\.SH "OPTIONS"(.*?)', manpage) is not None
1376
1377    def parse_man_page(self, manpage):
1378        options_section_regex = re.compile(r'\.SH "OPTIONS"(.*?)(\.SH|\Z)', re.DOTALL)
1379        options_section = re.search(options_section_regex, manpage).group(1)
1380
1381        options_parts_regex = re.compile(r"\.PP(.*?)\.RE", re.DOTALL)
1382        options_matched = re.search(options_parts_regex, options_section)
1383        add_diagnostic("Command is %r" % CMDNAME)
1384
1385        if options_matched is None:
1386            add_diagnostic("Unable to find options")
1387            if self.fallback(options_section):
1388                return True
1389            elif self.fallback2(options_section):
1390                return True
1391            return False
1392
1393        while options_matched is not None:
1394            data = options_matched.group(1)
1395            last_dotpp_index = data.rfind(".PP")
1396            if last_dotpp_index != -1:
1397                data = data[last_dotpp_index + 3 :]
1398
1399            data = remove_groff_formatting(data)
1400            data = data.split(".RS 4")
1401            if len(data) > 1:  # and len(data[1]) <= 300):
1402                optionName = data[0].strip()
1403
1404                if optionName.find("-") == -1:
1405                    add_diagnostic("%r doesn't contain '-' " % optionName)
1406                else:
1407                    optionName = unquote_double_quotes(optionName)
1408                    optionName = unquote_single_quotes(optionName)
1409                    optionDescription = data[1].strip().replace("\n", " ")
1410                    built_command(optionName, optionDescription)
1411
1412            else:
1413                add_diagnostic("Unable to split option from description")
1414                return False
1415
1416            options_section = options_section[options_matched.end() - 3 :]
1417            options_matched = re.search(options_parts_regex, options_section)
1418
1419    def fallback(self, options_section):
1420        add_diagnostic("Trying fallback")
1421        options_parts_regex = re.compile(r"\.TP( \d+)?(.*?)\.TP", re.DOTALL)
1422        options_matched = re.search(options_parts_regex, options_section)
1423        if options_matched is None:
1424            add_diagnostic("Still not found")
1425            return False
1426        while options_matched is not None:
1427            data = options_matched.group(2)
1428            data = remove_groff_formatting(data)
1429            data = data.strip()
1430            data = data.split("\n", 1)
1431            if len(data) > 1 and len(data[1].strip()) > 0:  # and len(data[1])<400):
1432                optionName = data[0].strip()
1433                if optionName.find("-") == -1:
1434                    add_diagnostic("%r doesn't contain '-'" % optionName)
1435                else:
1436                    optionName = unquote_double_quotes(optionName)
1437                    optionName = unquote_single_quotes(optionName)
1438                    optionDescription = data[1].strip().replace("\n", " ")
1439                    built_command(optionName, optionDescription)
1440            else:
1441                add_diagnostic("Unable to split option from description")
1442                return False
1443
1444            options_section = options_section[options_matched.end() - 3 :]
1445            options_matched = re.search(options_parts_regex, options_section)
1446        return True
1447
1448    def fallback2(self, options_section):
1449        add_diagnostic("Trying last chance fallback")
1450        ix_remover_regex = re.compile(r"\.IX.*")
1451        trailing_num_regex = re.compile("\\d+$")
1452        options_parts_regex = re.compile(r"\.IP (.*?)\.IP", re.DOTALL)
1453
1454        options_section = re.sub(ix_remover_regex, "", options_section)
1455        options_matched = re.search(options_parts_regex, options_section)
1456        if options_matched is None:
1457            add_diagnostic("Still (still!) not found")
1458            return False
1459        while options_matched is not None:
1460            data = options_matched.group(1)
1461
1462            data = remove_groff_formatting(data)
1463            data = data.strip()
1464            data = data.split("\n", 1)
1465            if len(data) > 1 and len(data[1].strip()) > 0:  # and len(data[1])<400):
1466                optionName = re.sub(trailing_num_regex, "", data[0].strip())
1467
1468                if "-" not in optionName:
1469                    add_diagnostic("%r doesn't contain '-'" % optionName)
1470                else:
1471                    optionName = optionName.strip()
1472                    optionName = unquote_double_quotes(optionName)
1473                    optionName = unquote_single_quotes(optionName)
1474                    optionDescription = data[1].strip().replace("\n", " ")
1475                    built_command(optionName, optionDescription)
1476            else:
1477                add_diagnostic("Unable to split option from description")
1478                return False
1479
1480            options_section = options_section[options_matched.end() - 3 :]
1481            options_matched = re.search(options_parts_regex, options_section)
1482        return True
1483
1484
1485class Type2ManParser(ManParser):
1486    def is_my_type(self, manpage):
1487        return compile_and_search(r"\.SH OPTIONS(.*?)", manpage) is not None
1488
1489    def parse_man_page(self, manpage):
1490        options_section_regex = re.compile(r"\.SH OPTIONS(.*?)(\.SH|\Z)", re.DOTALL)
1491        options_section = re.search(options_section_regex, manpage).group(1)
1492
1493        options_parts_regex = re.compile(
1494            r"\.[IT]P( \d+(\.\d)?i?)?(.*?)\.([IT]P|UNINDENT|UN|SH)", re.DOTALL
1495        )
1496        options_matched = re.search(options_parts_regex, options_section)
1497        add_diagnostic("Command is %r" % CMDNAME)
1498
1499        if options_matched is None:
1500            add_diagnostic("%r: Unable to find options" % self)
1501            return False
1502
1503        while options_matched is not None:
1504            data = options_matched.group(3)
1505            data = remove_groff_formatting(data)
1506            data = data.strip()
1507            data = data.split("\n", 1)
1508
1509            if len(data) > 1 and len(data[1].strip()) > 0:  # and len(data[1])<400):
1510                optionName = data[0].strip()
1511                if "-" not in optionName:
1512                    add_diagnostic("%r doesn't contain '-'" % optionName)
1513                else:
1514                    optionName = unquote_double_quotes(optionName)
1515                    optionName = unquote_single_quotes(optionName)
1516                    optionDescription = data[1].strip().replace("\n", " ")
1517                    built_command(optionName, optionDescription)
1518            else:
1519                add_diagnostic("Unable to split option from description")
1520
1521            options_section = options_section[options_matched.end() - 3 :]
1522            options_matched = re.search(options_parts_regex, options_section)
1523
1524
1525class Type3ManParser(ManParser):
1526    def is_my_type(self, manpage):
1527        return compile_and_search(r"\.SH DESCRIPTION(.*?)", manpage) is not None
1528
1529    def parse_man_page(self, manpage):
1530        options_section_regex = re.compile(r"\.SH DESCRIPTION(.*?)(\.SH|\Z)", re.DOTALL)
1531        options_section = re.search(options_section_regex, manpage).group(1)
1532
1533        options_parts_regex = re.compile(r"\.TP(.*?)\.TP", re.DOTALL)
1534        options_matched = re.search(options_parts_regex, options_section)
1535        add_diagnostic("Command is %r" % CMDNAME)
1536
1537        if options_matched is None:
1538            add_diagnostic("Unable to find options section")
1539            return False
1540
1541        while options_matched is not None:
1542            data = options_matched.group(1)
1543
1544            data = remove_groff_formatting(data)
1545            data = data.strip()
1546            data = data.split("\n", 1)
1547
1548            if len(data) > 1:  # and len(data[1])<400):
1549                optionName = data[0].strip()
1550                if optionName.find("-") == -1:
1551                    add_diagnostic("%r doesn't contain '-'" % optionName)
1552                else:
1553                    optionName = unquote_double_quotes(optionName)
1554                    optionName = unquote_single_quotes(optionName)
1555                    optionDescription = data[1].strip().replace("\n", " ")
1556                    built_command(optionName, optionDescription)
1557
1558            else:
1559                add_diagnostic("Unable to split option from description")
1560                return False
1561
1562            options_section = options_section[options_matched.end() - 3 :]
1563            options_matched = re.search(options_parts_regex, options_section)
1564
1565
1566class Type4ManParser(ManParser):
1567    def is_my_type(self, manpage):
1568        return compile_and_search(r"\.SH FUNCTION LETTERS(.*?)", manpage) is not None
1569
1570    def parse_man_page(self, manpage):
1571        options_section_regex = re.compile(
1572            r"\.SH FUNCTION LETTERS(.*?)(\.SH|\Z)", re.DOTALL
1573        )
1574        options_section = re.search(options_section_regex, manpage).group(1)
1575
1576        options_parts_regex = re.compile(r"\.TP(.*?)\.TP", re.DOTALL)
1577        options_matched = re.search(options_parts_regex, options_section)
1578        add_diagnostic("Command is %r" % CMDNAME)
1579
1580        if options_matched is None:
1581            print("Unable to find options section", file=sys.stderr)
1582            return False
1583
1584        while options_matched is not None:
1585            data = options_matched.group(1)
1586
1587            data = remove_groff_formatting(data)
1588            data = data.strip()
1589            data = data.split("\n", 1)
1590
1591            if len(data) > 1:  # and len(data[1])<400):
1592                optionName = data[0].strip()
1593                if optionName.find("-") == -1:
1594                    add_diagnostic("%r doesn't contain '-' " % optionName)
1595                else:
1596                    optionName = unquote_double_quotes(optionName)
1597                    optionName = unquote_single_quotes(optionName)
1598                    optionDescription = data[1].strip().replace("\n", " ")
1599                    built_command(optionName, optionDescription)
1600
1601            else:
1602                add_diagnostic("Unable to split option from description")
1603                return False
1604
1605            options_section = options_section[options_matched.end() - 3 :]
1606            options_matched = re.search(options_parts_regex, options_section)
1607
1608
1609class TypeScdocManParser(ManParser):
1610    def is_my_type(self, manpage):
1611        return (
1612            compile_and_search(
1613                r"\.(\\)(\") Generated by scdoc(.*?)\.SH OPTIONS(.*?)", manpage
1614            )
1615            is not None
1616        )
1617
1618    def parse_man_page(self, manpage):
1619        options_section_regex = re.compile(r"\.SH OPTIONS(.*?)\.SH", re.DOTALL)
1620        options_section_matched = re.search(options_section_regex, manpage)
1621        if options_section_matched is None:
1622            return False
1623        options_section = options_section_matched.group(1)
1624
1625        options_parts_regex = re.compile(r"(.*?)\.RE", re.DOTALL)
1626        options_matched = re.match(options_parts_regex, options_section)
1627        add_diagnostic("Command is %r" % CMDNAME)
1628
1629        if options_matched is None:
1630            add_diagnostic("%r: Unable to find options" % self)
1631            return False
1632
1633        while options_matched is not None:
1634            # Get first option and move options_section
1635            option = options_matched.group(1)
1636            options_section = options_section[options_matched.end() : :]
1637            options_matched = re.match(options_parts_regex, options_section)
1638
1639            option = remove_groff_formatting(option)
1640            option = option.split("\n")
1641
1642            # Crean option list
1643            option_clean = []
1644            for line in option:
1645                if line not in ["", ".P", ".RS 4"]:
1646                    option_clean.append(line)
1647
1648            # Shold be at least two lines
1649            if len(option_clean) < 2:
1650                add_diagnostic("Unable to split option from description")
1651                continue
1652
1653            # Name and description, others lines are ignored
1654            option_name = option_clean[0]
1655            option_description = option_clean[1]
1656
1657            if "-" not in option_name:
1658                add_diagnostic("%r doesn't contain '-'" % option_name)
1659
1660            option_name = unquote_double_quotes(option_name)
1661            option_name = unquote_single_quotes(option_name)
1662            built_command(option_name, option_description)
1663
1664        return True
1665
1666
1667class TypeDarwinManParser(ManParser):
1668    def is_my_type(self, manpage):
1669        return compile_and_search(r"\.S[hH] DESCRIPTION", manpage) is not None
1670
1671    def trim_groff(self, line):
1672        # Remove initial period
1673        if line.startswith("."):
1674            line = line[1:]
1675        # Skip leading groff crud
1676        while re.match(r"[A-Z][a-z]\s", line):
1677            line = line[3:]
1678
1679        # If the line ends with a space and then a period or comma, then erase the space
1680        # This hack handles lines of the form '.Ar projectname .'
1681        if line.endswith(" ,") or line.endswith(" ."):
1682            line = line[:-2] + line[-1]
1683        return line
1684
1685    def count_argument_dashes(self, line):
1686        # Determine how many dashes the line has using the following regex hack
1687        # Look for the start of a line, followed by a dot, then a sequence of
1688        # one or more dashes ('Fl')
1689        result = 0
1690        if line.startswith("."):
1691            line = line[4:]
1692            while line.startswith("Fl "):
1693                result = result + 1
1694                line = line[3:]
1695        return result
1696
1697    # Replace some groff escapes. There's a lot we don't bother to handle.
1698    def groff_replace_escapes(self, line):
1699        line = line.replace(".Nm", CMDNAME)
1700        line = line.replace("\\ ", " ")
1701        line = line.replace(r"\& ", "")
1702        line = line.replace(".Pp", "")
1703        return line
1704
1705    def is_option(self, line):
1706        return line.startswith(".It Fl")
1707
1708    def parse_man_page(self, manpage):
1709        got_something = False
1710        lines = manpage.splitlines()
1711        # Discard lines until we get to ".sh Description"
1712        while lines and not (
1713            lines[0].startswith(".Sh DESCRIPTION")
1714            or lines[0].startswith(".SH DESCRIPTION")
1715        ):
1716            lines.pop(0)
1717
1718        while lines:
1719            # Pop until we get to the next option
1720            while lines and not self.is_option(lines[0]):
1721                lines.pop(0)
1722
1723            if not lines:
1724                continue
1725
1726            # Get the line and clean it up
1727            line = lines.pop(0)
1728
1729            # Try to guess how many dashes this argument has
1730            dash_count = self.count_argument_dashes(line)
1731
1732            line = self.groff_replace_escapes(line)
1733            line = self.trim_groff(line)
1734            line = line.strip()
1735            if not line:
1736                continue
1737
1738            # Extract the name
1739            name = line.split(None, 2)[0]
1740
1741            # Extract the description
1742            desc_lines = []
1743            while lines and not self.is_option(lines[0]):
1744                line = lines.pop(0).strip()
1745                # Ignore comments
1746                if line.startswith(r".\""):
1747                    continue
1748                if line.startswith("."):
1749                    line = self.groff_replace_escapes(line)
1750                    line = self.trim_groff(line).strip()
1751                if line:
1752                    desc_lines.append(line)
1753            desc = " ".join(desc_lines)
1754
1755            if name == "-":
1756                # Skip double -- arguments
1757                continue
1758            elif len(name) > 1:
1759                # Output the command
1760                built_command(("-" * dash_count) + name, desc)
1761                got_something = True
1762            elif len(name) == 1:
1763                built_command("-" + name, desc)
1764                got_something = True
1765
1766        return got_something
1767
1768
1769class TypeDeroffManParser(ManParser):
1770    def is_my_type(self, manpage):
1771        return True  # We're optimists
1772
1773    def is_option(self, line):
1774        return line.startswith("-")
1775
1776    def could_be_description(self, line):
1777        return len(line) > 0 and not line.startswith("-")
1778
1779    def parse_man_page(self, manpage):
1780        d = Deroffer()
1781        d.deroff(manpage)
1782        output = d.get_output()
1783        lines = output.split("\n")
1784
1785        got_something = False
1786
1787        # Discard lines until we get to DESCRIPTION or OPTIONS
1788        while lines and not (
1789            lines[0].startswith("DESCRIPTION")
1790            or lines[0].startswith("OPTIONS")
1791            or lines[0].startswith("COMMAND OPTIONS")
1792        ):
1793            lines.pop(0)
1794
1795        # Look for BUGS and stop there
1796        for idx in range(len(lines)):
1797            line = lines[idx]
1798            if line.startswith("BUGS"):
1799                # Drop remaining elements
1800                lines[idx:] = []
1801                break
1802
1803        while lines:
1804            # Pop until we get to the next option
1805            while lines and not self.is_option(lines[0]):
1806                line = lines.pop(0)
1807
1808            if not lines:
1809                continue
1810
1811            options = lines.pop(0)
1812
1813            # Pop until we get to either an empty line or a line starting with -
1814            description = ""
1815            while lines and self.could_be_description(lines[0]):
1816                if description:
1817                    description += " "
1818                description += lines.pop(0)
1819
1820            built_command(options, description)
1821            got_something = True
1822
1823        return got_something
1824
1825
1826# Return whether the file at the given path is overwritable
1827# Raises IOError if it cannot be opened
1828def file_is_overwritable(path):
1829    result = False
1830    file = open(path, "r", encoding="utf-8")
1831    for line in file:
1832        # Skip leading empty lines
1833        line = line.strip()
1834        if not line:
1835            continue
1836
1837        # We look in the initial run of lines that start with #
1838        if not line.startswith("#"):
1839            break
1840
1841        # See if this contains the magic word
1842        if "Autogenerated" in line:
1843            result = True
1844            break
1845
1846    file.close()
1847    return result
1848
1849
1850# Remove any and all autogenerated completions in the given directory
1851def cleanup_autogenerated_completions_in_directory(dir):
1852    try:
1853        for filename in os.listdir(dir):
1854            # Skip non .fish files
1855            if not filename.endswith(".fish"):
1856                continue
1857            path = os.path.join(dir, filename)
1858            cleanup_autogenerated_file(path)
1859    except OSError:
1860        return False
1861
1862
1863# Delete the file if it is autogenerated
1864def cleanup_autogenerated_file(path):
1865    try:
1866        if file_is_overwritable(path):
1867            os.remove(path)
1868    except (OSError, IOError):
1869        pass
1870
1871
1872def parse_manpage_at_path(manpage_path, output_directory):
1873    # Return if CMDNAME is in 'ignoredcommands'
1874    ignoredcommands = [
1875        "cc",
1876        "g++",
1877        "gcc",
1878        "c++",
1879        "cpp",
1880        "emacs",
1881        "gprof",
1882        "wget",
1883        "ld",
1884        "awk",
1885    ]
1886    if CMDNAME in ignoredcommands:
1887        return
1888
1889    # Ignore some commands' gazillion man pages
1890    # for subcommands - especially things we already have
1891    ignored_prefixes = [
1892        "bundle-",
1893        "cargo-",
1894        "ffmpeg-",
1895        "flatpak-",
1896        "gcloud_",
1897        "git-",
1898        "jj-",
1899        "npm-",
1900        "openssl-",
1901        "ostree-",
1902        "perf-",
1903        "perl",
1904        "pip-",
1905        "zsh",
1906    ]
1907    for prefix in ignored_prefixes:
1908        if CMDNAME.startswith(prefix):
1909            return
1910
1911    # Clear diagnostics
1912    global diagnostic_indent
1913    diagnostic_output[:] = []
1914    diagnostic_indent = 0
1915
1916    # Set up some diagnostics
1917    add_diagnostic("Considering " + manpage_path)
1918    diagnostic_indent += 1
1919
1920    if manpage_path.endswith(".gz"):
1921        fd = gzip.open(manpage_path, "r")
1922        manpage = fd.read()
1923        manpage = manpage.decode("latin-1")
1924    elif manpage_path.endswith(".bz2"):
1925        fd = bz2.BZ2File(manpage_path, "r")
1926        manpage = fd.read()
1927        manpage = manpage.decode("latin-1")
1928    elif manpage_path.endswith(".xz") or manpage_path.endswith(".lzma"):
1929        if not lzma_available:
1930            return
1931        fd = lzma.LZMAFile(str(manpage_path), "r")
1932        manpage = fd.read()
1933        manpage = manpage.decode("latin-1")
1934    elif manpage_path.endswith((".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9")):
1935        fd = open(manpage_path, "r", encoding="latin-1")
1936        manpage = fd.read()
1937    else:
1938        return
1939    fd.close()
1940
1941    manpage = str(manpage)
1942
1943    # Ignore the millions of links to BUILTIN(1)
1944    if "BUILTIN 1" in manpage or "builtin.1" in manpage:
1945        return
1946
1947    # Clear the output list
1948    built_command_output[:] = []
1949    global already_output_completions
1950    already_output_completions = {}
1951
1952    if DEROFF_ONLY:
1953        parsers = [TypeDeroffManParser()]
1954    else:
1955        parsers = [
1956            TypeScdocManParser(),
1957            Type1ManParser(),
1958            Type2ManParser(),
1959            Type4ManParser(),
1960            Type3ManParser(),
1961            TypeDarwinManParser(),
1962            TypeDeroffManParser(),
1963        ]
1964    parsersToTry = [p for p in parsers if p.is_my_type(manpage)]
1965
1966    success = False
1967    for parser in parsersToTry:
1968        add_diagnostic("Trying %s" % parser.__class__.__name__)
1969        diagnostic_indent += 1
1970        success = parser.parse_man_page(manpage)
1971        diagnostic_indent -= 1
1972        # Make sure empty files aren't reported as success
1973        if not built_command_output:
1974            success = False
1975        if success:
1976            PARSER_INFO.setdefault(parser.__class__.__name__, []).append(CMDNAME)
1977            break
1978
1979    if success:
1980        if WRITE_TO_STDOUT:
1981            output_file = sys.stdout
1982        else:
1983            fullpath = os.path.join(output_directory, CMDNAME + ".fish")
1984            try:
1985                output_file = open(fullpath, "w", encoding="utf-8")
1986            except IOError as err:
1987                add_diagnostic(
1988                    "Unable to open file '%s': error(%d): %s"
1989                    % (fullpath, err.errno, err.strerror)
1990                )
1991                return False
1992
1993        # Output the magic word Autogenerated so we can tell if we can overwrite this
1994        built_command_output.insert(
1995            0, "# " + CMDNAME + "\n# Autogenerated from man page " + manpage_path
1996        )
1997        # built_command_output.insert(2, "# using " + parser.__class__.__name__) # XXX MISATTRIBUTES THE CULPABLE PARSER! Was really using Type2 but reporting TypeDeroffManParser
1998
1999        for line in built_command_output:
2000            output_file.write(line)
2001            output_file.write("\n")
2002        output_file.write("\n")
2003        add_diagnostic(manpage_path + " parsed successfully")
2004        if output_file != sys.stdout:
2005            output_file.close()
2006    else:
2007        parser_names = ", ".join(p.__class__.__name__ for p in parsersToTry)
2008        # add_diagnostic('%s contains no options or is unparsable' % manpage_path, BRIEF_VERBOSE)
2009        add_diagnostic(
2010            "%s contains no options or is unparsable (tried parser %s)"
2011            % (manpage_path, parser_names),
2012            BRIEF_VERBOSE,
2013        )
2014
2015    return success
2016
2017
2018def parse_and_output_man_pages(paths, output_directory, show_progress):
2019    global diagnostic_indent, CMDNAME
2020    paths.sort()
2021    total_count = len(paths)
2022    successful_count, index = 0, 0
2023    padding_len = len(str(total_count))
2024    last_progress_string_length = 0
2025    if show_progress and not WRITE_TO_STDOUT:
2026        print(
2027            "Parsing man pages and writing completions to {0}".format(output_directory)
2028        )
2029
2030    man_page_suffixes = set([os.path.splitext(m)[1][1:] for m in paths])
2031    lzma_xz_occurs = "xz" in man_page_suffixes or "lzma" in man_page_suffixes
2032    if lzma_xz_occurs and not lzma_available:
2033        add_diagnostic(
2034            'At least one man page is compressed with lzma or xz, but the "lzma" module is not available.'
2035            " Any man page compressed with either will be skipped.",
2036            NOT_VERBOSE,
2037        )
2038        flush_diagnostics(sys.stderr)
2039
2040    for manpage_path in paths:
2041        index += 1
2042
2043        # Get the "base" command, e.g. mkfs.xfs.8.gz -> mkfs.xfs
2044        man_file_name = os.path.basename(manpage_path)
2045        # 1. strip the optional compressor suffix
2046        CMDNAME, extension = os.path.splitext(man_file_name)
2047        # 2. strip mandatory section nu:ber
2048        if CMDNAME.endswith((".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9")):
2049            CMDNAME, extension = os.path.splitext(CMDNAME)
2050        # 3. XXX strip optional version(?) number?
2051        # see comment above `already_output_completions = {}` line
2052
2053        # Show progress if we're doing that
2054        if show_progress:
2055            progress_str = "  {0} / {1} : {2}".format(
2056                (str(index).rjust(padding_len)), total_count, man_file_name
2057            )
2058            # Pad on the right with spaces so we overwrite whatever we wrote last time
2059            padded_progress_str = progress_str.ljust(last_progress_string_length)
2060            last_progress_string_length = len(progress_str)
2061            sys.stdout.write("\r{0}\r".format(padded_progress_str))
2062            sys.stdout.flush()
2063
2064        try:
2065            if parse_manpage_at_path(manpage_path, output_directory):
2066                successful_count += 1
2067        except IOError:
2068            diagnostic_indent = 0
2069            add_diagnostic("Cannot open " + manpage_path)
2070        except (KeyboardInterrupt, SystemExit):
2071            raise
2072        except:
2073            add_diagnostic(
2074                "Error parsing %s: %s" % (manpage_path, sys.exc_info()[0]),
2075                BRIEF_VERBOSE,
2076            )
2077            flush_diagnostics(sys.stderr)
2078            traceback.print_exc(file=sys.stderr)
2079        flush_diagnostics(sys.stderr)
2080    print("")  # Newline after loop
2081    add_diagnostic(
2082        "Successfully parsed %d / %d pages" % (successful_count, total_count),
2083        BRIEF_VERBOSE,
2084    )
2085    flush_diagnostics(sys.stderr)
2086
2087
2088def get_paths_from_man_locations():
2089    # Return all the paths to man(1) and man(8) files in the manpath
2090
2091    parent_paths = []
2092
2093    # Most (GNU, macOS, Haiku) modern implementations of man support being called with `--path`.
2094    # Traditional implementations require a second `manpath` program: examples include FreeBSD and Solaris.
2095    # Prefer an external program first because these programs return a superset of the $MANPATH variable.
2096    for prog in [["man", "--path"], ["manpath"]]:
2097        try:
2098            output = subprocess.check_output(prog, stderr=DEVNULL)
2099            output = output.decode("latin-1")
2100            parent_paths = output.strip().split(":")
2101            break
2102        except (OSError, subprocess.CalledProcessError):
2103            continue
2104    # If we can't have the OS interpret $MANPATH, just use it as-is (gulp).
2105    if not parent_paths and os.getenv("MANPATH"):
2106        parent_paths = os.getenv("MANPATH").strip().split(":")
2107    # Fallback: With mandoc (OpenBSD, embedded Linux) and NetBSD man, the only way to get the default manpath is by reading /etc.
2108    if not parent_paths:
2109        try:
2110            with open("/etc/man.conf", "r") as file:
2111                data = file.read()
2112                for key in ["MANPATH", "_default"]:
2113                    # novermin
2114                    for match in re.findall(r"^%s\s+(.*)$" % key, data, re.I | re.M):
2115                        parent_paths.append(match)
2116        except FileNotFoundError:
2117            pass
2118    # Fallback: hard-code some common paths. These should be likely for FHS Linux distros, BSDs, and macOS.
2119    if not parent_paths:
2120        parent_paths = ["/usr/share/man", "/usr/local/man", "/usr/local/share/man"]
2121        print(
2122            "Unable to get the manpath, falling back to %s." % ":".join(parent_paths),
2123            "Explicitly set $MANPATH to fix this error.",
2124            file=sys.stderr,
2125        )
2126
2127    result = []
2128    for parent_path in parent_paths:
2129        for section in ["man1", "man6", "man8"]:
2130            directory_path = os.path.join(parent_path, section)
2131            try:
2132                names = os.listdir(directory_path)
2133            except OSError:
2134                names = []
2135            names.sort()
2136            for name in names:
2137                result.append(os.path.join(directory_path, name))
2138    return result
2139
2140
2141if __name__ == "__main__":
2142    script_name = sys.argv[0]
2143    parser = argparse.ArgumentParser(
2144        description="create_manpage_completions: Generate fish-shell completions from manpages"
2145    )
2146    parser.add_argument(
2147        "-c",
2148        "--cleanup-in",
2149        type=str,
2150        help="Directories to clean up in",
2151        action="append",
2152    )
2153    parser.add_argument(
2154        "-d",
2155        "--directory",
2156        type=str,
2157        help="The directory to save the completions in",
2158    )
2159    parser.add_argument(
2160        "-k",
2161        "--keep",
2162        help="Whether to keep files in the target directory",
2163        action="store_true",
2164    )
2165    parser.add_argument(
2166        "-m",
2167        "--manpath",
2168        help="Whether to use manpath",
2169        action="store_true",
2170    )
2171    parser.add_argument(
2172        "-p",
2173        "--progress",
2174        help="Whether to show progress",
2175        action="store_true",
2176    )
2177    parser.add_argument(
2178        "-s",
2179        "--stdout",
2180        help="Write the completions to stdout",
2181        action="store_true",
2182    )
2183    parser.add_argument(
2184        "-v",
2185        "--verbose",
2186        type=int,
2187        choices=[0, 1, 2],
2188        help="The level of debug output to show",
2189    )
2190    parser.add_argument(
2191        "-z",
2192        "--deroff-only",
2193        help="Whether to just deroff",
2194        action="store_true",
2195    )
2196    parser.add_argument("file_paths", type=str, nargs="*")
2197
2198    args = parser.parse_args()
2199
2200    if args.verbose:
2201        VERBOSITY = args.verbose
2202    if args.stdout:
2203        WRITE_TO_STDOUT = True
2204    if args.deroff_only:
2205        DEROFF_ONLY = True
2206    if args.keep:
2207        KEEP_FILES = True
2208    if args.manpath:
2209        # Fetch all man1 and man8 files from the manpath or man.conf
2210        args.file_paths.extend(get_paths_from_man_locations())
2211
2212    # Directories within which we will clean up autogenerated completions
2213    # This script originally wrote completions into ~/.config/fish/completions
2214    # Now it writes them into a separate directory
2215    if args.cleanup_in:
2216        for cleanup_dir in args.cleanup_in:
2217            cleanup_autogenerated_completions_in_directory(cleanup_dir)
2218
2219    if not args.file_paths:
2220        if args.manpath:
2221            print("No paths specified and I can't make sense of your MANPATH")
2222            print("Please either give paths or set $MANPATH and try again")
2223        else:
2224            print("No paths specified")
2225        sys.exit(0)
2226
2227    if not args.stdout and not args.directory:
2228        # Default to ~/.cache/fish/generated_completions
2229        # Create it if it doesn't exist
2230        xdg_cache_home = os.getenv("XDG_CACHE_HOME", "~/.cache")
2231        args.directory = os.path.expanduser(
2232            xdg_cache_home + "/fish/generated_completions/"
2233        )
2234    if not args.stdout:
2235        try:
2236            os.makedirs(args.directory)
2237        except OSError as e:
2238            if e.errno != errno.EEXIST:
2239                raise
2240
2241    if not args.stdout and not args.keep:
2242        # Remove old generated files
2243        cleanup_autogenerated_completions_in_directory(args.directory)
2244
2245    parse_and_output_man_pages(args.file_paths, args.directory, args.progress)