+/TECS
git clone https://git.julia.blue/+/TECS
c8053ff
v1.0
1class Tokenizer : 2symbols = "{}()[].,;+-*/&|~<>=" 3keywords = ["do" ,"if" ,"var" ,"int" ,"let" ,"null" ,"this" ,"else" ,"char" ,"true" ,"void" ,"class" ,"field" ,"false" ,"while" ,"static" ,"return" ,"method" ,"boolean" ,"function" ,"constructor" ] 4 5def __init__ (self ,fileName ): 6self .reader = open (fileName ) 7self .token = "" 8self .tokenType = "" 9self .line = self .reader .readline ().lstrip (' \t' ) 10 11def parseInt (self ): 12i = 1 13while self .line [i ].isdigit (): 14i += 1 15self .tokenType = "integerConstant" 16self .token = self .line [:i ] 17self .line = self .line [i :] 18 19def parseString (self ): 20self .tokenType = "stringConstant" 21self .token = self .line .split ('"' )[1 ] 22self .line = '"' .join (self .line .split ('"' )[2 :]) 23 24def parseSymbol (self ): 25self .tokenType = "symbol" 26self .token = self .line [0 ] 27self .line = self .line [1 :] 28 29def parseKeyword (self ): 30for i in range (2 ,max (len (self .line ),11 )): 31if self .line [:i ]in self .keywords and not self .line [i ].isalpha (): 32self .tokenType = "keyword" 33self .token = self .line [:i ] 34self .line = self .line [i :] 35return True 36return False 37 38def parseIdentifier (self ): 39i = 1 40while self .line [i ].isalnum ()or self .line [i ]== '_' : 41i += 1 42self .tokenType = "identifier" 43self .token = self .line [:i ] 44self .line = self .line [i :] 45 46def nextToken (self ): 47while True : 48if self .line == "\n" or self .line [:2 ]== "//" :# skip single line comments and empty lines 49self .line = self .reader .readline ().lstrip (' \t' ) 50elif self .line [:2 ]== "/*" :# skip everything until end of block/api comment is found 51while "*/" not in self .line : 52self .line = self .reader .readline ().lstrip (' \t' ) 53if self .line == "" : 54return False 55self .line = self .line [self .line .find ("*/" )+ 2 :] 56elif self .line == "" :# readline() only reads empty string at EOF 57return False 58else : 59return True 60 61def advance (self ): 62self .line = self .line .lstrip (' \t' ) 63if not self .nextToken (): 64return False 65if self .line [0 ].isdigit (): 66self .parseInt () 67elif self .line [0 ]in self .symbols : 68self .parseSymbol () 69elif self .line [0 ]== '"' : 70self .parseString () 71else : 72if not self .parseKeyword (): 73self .parseIdentifier () 74return True