+/TECS

git clone https://git.julia.blue/+/TECS

P1n3appl3Add break statementd69793d

master
2.8 KiB78 linesraw
1class Tokenizer:
2    symbols = "{}()[].,;+-*/%&|~<>="
3    keywords = ["do", "if", "var", "int", "let", "null", "this", "else", "char", "true", "void", "class", "field", "false", "while", "static", "return", "method", "boolean", "function", "constructor", "break"]
4
5    def __init__(self, fileName):
6        self.reader = open(fileName)
7        self.token = ""
8        self.tokenType = ""
9        self.line = self.reader.readline().lstrip(' \t')
10
11    def parseInt(self):
12        i = 1
13        while self.line[i].isdigit():
14            i += 1
15        self.tokenType = "integerConstant"
16        self.token = self.line[:i]
17        self.line = self.line[i:]
18
19    def parseString(self):
20        self.tokenType = "stringConstant"
21        self.token = self.line.split('"')[1]
22        self.line = '"'.join(self.line.split('"')[2:])
23
24    def parseSymbol(self):
25        self.tokenType = "symbol"
26        self.token = self.line[0]
27        self.line = self.line[1:]
28        if len(self.line) > 0 and self.token in "~<>=" and self.line[0] == '=':
29            self.token += self.line[0]
30            self.line = self.line[1:]
31
32
33    def parseKeyword(self):
34        for i in range(2, max(len(self.line), 11)):
35            if self.line[:i] in self.keywords and not self.line[i].isalpha():
36                self.tokenType = "keyword"
37                self.token = self.line[:i]
38                self.line = self.line[i:]
39                return True
40        return False
41
42    def parseIdentifier(self):
43        i = 1
44        while self.line[i].isalnum() or self.line[i] == '_':
45            i += 1
46        self.tokenType = "identifier"
47        self.token = self.line[:i]
48        self.line = self.line[i:]
49
50    def nextToken(self):
51        while True:
52            if self.line == "\n" or self.line[:2] == "//":  # skip single line comments and empty lines
53                self.line = self.reader.readline().lstrip(' \t')
54            elif self.line[:2] == "/*":  # skip everything until end of block/api comment is found
55                while "*/" not in self.line:
56                    self.line = self.reader.readline().lstrip(' \t')
57                    if self.line == "":
58                        return False
59                self.line = self.line[self.line.find("*/") + 2:]
60            elif self.line == "":  # readline() only reads empty string at EOF
61                return False
62            else:
63                return True
64
65    def advance(self):
66        self.line = self.line.lstrip(' \t')
67        if not self.nextToken():
68            return False
69        if self.line[0].isdigit():
70            self.parseInt()
71        elif self.line[0] in self.symbols:
72            self.parseSymbol()
73        elif self.line[0] == '"':
74            self.parseString()
75        else:
76            if not self.parseKeyword():
77                self.parseIdentifier()
78        return True