+/TECS
git clone https://git.julia.blue/+/TECS
d69793d
master
1class Tokenizer : 2symbols = "{}()[].,;+-*/%&|~<>=" 3keywords = ["do" ,"if" ,"var" ,"int" ,"let" ,"null" ,"this" ,"else" ,"char" ,"true" ,"void" ,"class" ,"field" ,"false" ,"while" ,"static" ,"return" ,"method" ,"boolean" ,"function" ,"constructor" ,"break" ] 4 5def __init__ (self ,fileName ): 6self .reader = open (fileName ) 7self .token = "" 8self .tokenType = "" 9self .line = self .reader .readline ().lstrip (' \t' ) 10 11def parseInt (self ): 12i = 1 13while self .line [i ].isdigit (): 14i += 1 15self .tokenType = "integerConstant" 16self .token = self .line [:i ] 17self .line = self .line [i :] 18 19def parseString (self ): 20self .tokenType = "stringConstant" 21self .token = self .line .split ('"' )[1 ] 22self .line = '"' .join (self .line .split ('"' )[2 :]) 23 24def parseSymbol (self ): 25self .tokenType = "symbol" 26self .token = self .line [0 ] 27self .line = self .line [1 :] 28if len (self .line )> 0 and self .token in "~<>=" and self .line [0 ]== '=' : 29self .token += self .line [0 ] 30self .line = self .line [1 :] 31 32 33def parseKeyword (self ): 34for i in range (2 ,max (len (self .line ),11 )): 35if self .line [:i ]in self .keywords and not self .line [i ].isalpha (): 36self .tokenType = "keyword" 37self .token = self .line [:i ] 38self .line = self .line [i :] 39return True 40return False 41 42def parseIdentifier (self ): 43i = 1 44while self .line [i ].isalnum ()or self .line [i ]== '_' : 45i += 1 46self .tokenType = "identifier" 47self .token = self .line [:i ] 48self .line = self .line [i :] 49 50def nextToken (self ): 51while True : 52if self .line == "\n" or self .line [:2 ]== "//" :# skip single line comments and empty lines 53self .line = self .reader .readline ().lstrip (' \t' ) 54elif self .line [:2 ]== "/*" :# skip everything until end of block/api comment is found 55while "*/" not in self .line : 56self .line = self .reader .readline ().lstrip (' \t' ) 57if self .line == "" : 58return False 59self .line = self .line [self .line .find ("*/" )+ 2 :] 60elif self .line == "" :# readline() only reads empty string at EOF 61return False 62else : 63return True 64 65def advance (self ): 66self .line = self .line .lstrip (' \t' ) 67if not self .nextToken (): 68return False 69if self .line [0 ].isdigit (): 70self .parseInt () 71elif self .line [0 ]in self .symbols : 72self .parseSymbol () 73elif self .line [0 ]== '"' : 74self .parseString () 75else : 76if not self .parseKeyword (): 77self .parseIdentifier () 78return True