+/TECS

git clone https://git.julia.blue/+/TECS

P1n3appl3Finish project 11c8053ff

v1.0
2.6 KiB74 linesraw
1class Tokenizer:
2    symbols = "{}()[].,;+-*/&|~<>="
3    keywords = ["do", "if", "var", "int", "let", "null", "this", "else", "char", "true", "void", "class", "field", "false", "while", "static", "return", "method", "boolean", "function", "constructor"]
4
5    def __init__(self, fileName):
6        self.reader = open(fileName)
7        self.token = ""
8        self.tokenType = ""
9        self.line = self.reader.readline().lstrip(' \t')
10
11    def parseInt(self):
12        i = 1
13        while self.line[i].isdigit():
14            i += 1
15        self.tokenType = "integerConstant"
16        self.token = self.line[:i]
17        self.line = self.line[i:]
18
19    def parseString(self):
20        self.tokenType = "stringConstant"
21        self.token = self.line.split('"')[1]
22        self.line = '"'.join(self.line.split('"')[2:])
23
24    def parseSymbol(self):
25        self.tokenType = "symbol"
26        self.token = self.line[0]
27        self.line = self.line[1:]
28
29    def parseKeyword(self):
30        for i in range(2, max(len(self.line), 11)):
31            if self.line[:i] in self.keywords and not self.line[i].isalpha():
32                self.tokenType = "keyword"
33                self.token = self.line[:i]
34                self.line = self.line[i:]
35                return True
36        return False
37
38    def parseIdentifier(self):
39        i = 1
40        while self.line[i].isalnum() or self.line[i] == '_':
41            i += 1
42        self.tokenType = "identifier"
43        self.token = self.line[:i]
44        self.line = self.line[i:]
45
46    def nextToken(self):
47        while True:
48            if self.line == "\n" or self.line[:2] == "//":  # skip single line comments and empty lines
49                self.line = self.reader.readline().lstrip(' \t')
50            elif self.line[:2] == "/*":  # skip everything until end of block/api comment is found
51                while "*/" not in self.line:
52                    self.line = self.reader.readline().lstrip(' \t')
53                    if self.line == "":
54                        return False
55                self.line = self.line[self.line.find("*/") + 2:]
56            elif self.line == "":  # readline() only reads empty string at EOF
57                return False
58            else:
59                return True
60
61    def advance(self):
62        self.line = self.line.lstrip(' \t')
63        if not self.nextToken():
64            return False
65        if self.line[0].isdigit():
66            self.parseInt()
67        elif self.line[0] in self.symbols:
68            self.parseSymbol()
69        elif self.line[0] == '"':
70            self.parseString()
71        else:
72            if not self.parseKeyword():
73                self.parseIdentifier()
74        return True