Module lexer
Expand source code
#!/usr/bin/env python
from __future__ import absolute_import
from __future__ import print_function
import re
# ------------------------------------------------------------------------
#
class _LexIter():
def __init__(self):
cnt = 0;
def lexiter(self, pos, tokens, strx):
#print strx[pos:]
for bb, cc, dd in tokens:
#print bb, cc
mmm = dd.match(strx, pos)
if mmm:
#print mmm.end() - mmm.start(), strx[mmm.start():mmm.end()]
tt = bb, mmm, strx[mmm.start():mmm.end()]
return tt
return None;
class Lexer():
def __init__(self, data, stack, tokens):
# Pre-compile tokens
cnt = 0;
while True:
try:
tokens[cnt][2] = re.compile(tokens[cnt][1])
except: break
cnt += 1
# print tokens
#for xx in tokens:
# print "token:", xx
lexiter = _LexIter()
lastpos = 0; pos = 0; lenx = len(data)
while True:
if pos >= lenx:
break;
tt = lexiter.lexiter(pos, tokens, data)
if tt == None: break
mmm = tt[1]
if mmm:
# skip token
pos = mmm.end()
#print tt[1], "'" + data[mmm.start():mmm.end()] + "' - ",
#print "'" + data[mmm.start():mmm.end()] + "' - ",
stack.push(tt)
else:
pos += 1 # step to next
if __name__ == "__main__":
print("This module was not meant to operate as main.")
# EOF
Classes
class Lexer (data, stack, tokens)
-
Expand source code
class Lexer(): def __init__(self, data, stack, tokens): # Pre-compile tokens cnt = 0; while True: try: tokens[cnt][2] = re.compile(tokens[cnt][1]) except: break cnt += 1 # print tokens #for xx in tokens: # print "token:", xx lexiter = _LexIter() lastpos = 0; pos = 0; lenx = len(data) while True: if pos >= lenx: break; tt = lexiter.lexiter(pos, tokens, data) if tt == None: break mmm = tt[1] if mmm: # skip token pos = mmm.end() #print tt[1], "'" + data[mmm.start():mmm.end()] + "' - ", #print "'" + data[mmm.start():mmm.end()] + "' - ", stack.push(tt) else: pos += 1 # step to next