Code cleanup.

parent f9a34e5e
...@@ -54,6 +54,7 @@ cdef extern from "../c/bisondynlib.h": ...@@ -54,6 +54,7 @@ cdef extern from "../c/bisondynlib.h":
#int bisondynlib_build(char *libName, char *includedir) #int bisondynlib_build(char *libName, char *includedir)
# Definitions for variadic functions (e.g. py_callback).
cdef extern from "stdarg.h": cdef extern from "stdarg.h":
ctypedef struct va_list: ctypedef struct va_list:
pass pass
...@@ -163,7 +164,7 @@ cdef class ParserEngine: ...@@ -163,7 +164,7 @@ cdef class ParserEngine:
- calling the entry point - calling the entry point
- closing the library - closing the library
Makes direct calls to the platform-dependent routines in Makes direct calls to the platform-dependent routines in
bisondynlib-[linux|windows].c bisondynlib-[linux|windows].c
""" """
cdef object parser cdef object parser
...@@ -246,12 +247,12 @@ cdef class ParserEngine: ...@@ -246,12 +247,12 @@ cdef class ParserEngine:
cdef char *libFilename cdef char *libFilename
cdef char *err cdef char *err
cdef void *handle cdef void *handle
# convert python filename string to c string # convert python filename string to c string
libFilename = PyString_AsString(self.libFilename_py) libFilename = PyString_AsString(self.libFilename_py)
parser = self.parser parser = self.parser
if parser.verbose: if parser.verbose:
print "Opening library %s" % self.libFilename_py print "Opening library %s" % self.libFilename_py
handle = bisondynlib_open(libFilename) handle = bisondynlib_open(libFilename)
...@@ -260,19 +261,17 @@ cdef class ParserEngine: ...@@ -260,19 +261,17 @@ cdef class ParserEngine:
if err: if err:
printf("ParserEngine.openLib: error '%s'\n", err) printf("ParserEngine.openLib: error '%s'\n", err)
return return
# extract symbols # extract symbols
self.libHash = bisondynlib_lookup_hash(handle) self.libHash = bisondynlib_lookup_hash(handle)
if parser.verbose: if parser.verbose:
print "Successfully loaded library" print "Successfully loaded library"
#@-node:openLib
#@+node:buildLib
def buildLib(self): def buildLib(self):
""" """
Creates the parser engine lib Creates the parser engine lib
This consists of: This consists of:
1. Ripping the tokens list, precedences, start target, handler docstrings 1. Ripping the tokens list, precedences, start target, handler docstrings
and lex script from this Parser instance's attribs and methods and lex script from this Parser instance's attribs and methods
...@@ -281,11 +280,11 @@ cdef class ParserEngine: ...@@ -281,11 +280,11 @@ cdef class ParserEngine:
4. Compiling the C files, and link into a dynamic lib 4. Compiling the C files, and link into a dynamic lib
""" """
cdef char *incdir cdef char *incdir
# ------------------------------------------------- # -------------------------------------------------
# rip the pertinent grammar specs from parser class # rip the pertinent grammar specs from parser class
parser = self.parser parser = self.parser
# get target handler methods, in the order of appearance in the source # get target handler methods, in the order of appearance in the source
# file. # file.
attribs = dir(parser) attribs = dir(parser)
...@@ -295,7 +294,7 @@ cdef class ParserEngine: ...@@ -295,7 +294,7 @@ cdef class ParserEngine:
method = getattr(parser, a) method = getattr(parser, a)
gHandlers.append(method) gHandlers.append(method)
gHandlers.sort(cmpLines) gHandlers.sort(cmpLines)
# get start symbol, tokens, precedences, lex script # get start symbol, tokens, precedences, lex script
gStart = parser.start gStart = parser.start
gTokens = parser.tokens gTokens = parser.tokens
...@@ -315,7 +314,7 @@ cdef class ParserEngine: ...@@ -315,7 +314,7 @@ cdef class ParserEngine:
f = open(buildDirectory + parser.bisonFile, "w") f = open(buildDirectory + parser.bisonFile, "w")
write = f.write write = f.write
writelines = f.writelines writelines = f.writelines
# grammar file prologue # grammar file prologue
write("\n".join([ write("\n".join([
"%{", "%{",
...@@ -366,7 +365,7 @@ cdef class ParserEngine: ...@@ -366,7 +365,7 @@ cdef class ParserEngine:
options = options.strip() options = options.strip()
tmp = [] tmp = []
#print "options = %s" % repr(options) #print "options = %s" % repr(options)
opts = options.split("|") opts = options.split("|")
#print "opts = %s" % repr(opts) #print "opts = %s" % repr(opts)
...@@ -374,15 +373,15 @@ cdef class ParserEngine: ...@@ -374,15 +373,15 @@ cdef class ParserEngine:
#print "r = <%s>" % r #print "r = <%s>" % r
opts1 = re.split(r, " " + options) opts1 = re.split(r, " " + options)
#print "opts1 = %s" % repr(opts1) #print "opts1 = %s" % repr(opts1)
for o in opts1: for o in opts1:
o = o.strip() o = o.strip()
tmp.append(reSpaces.split(o)) tmp.append(reSpaces.split(o))
options = tmp options = tmp
rules.append((target, options)) rules.append((target, options))
# and render rules to grammar file # and render rules to grammar file
for rule in rules: for rule in rules:
try: try:
...@@ -409,19 +408,19 @@ cdef class ParserEngine: ...@@ -409,19 +408,19 @@ cdef class ParserEngine:
i = i - 1 i = i - 1
break # hack for rules using '%prec' break # hack for rules using '%prec'
args.append('"%s", $%d' % (option[i], i+1)) args.append('"%s", $%d' % (option[i], i+1))
# now, we have the correct terms count # now, we have the correct terms count
action = action % (i + 1) action = action % (i + 1)
# assemble the full rule + action, ad to list # assemble the full rule + action, ad to list
action = action + ",\n " action = action + ",\n "
action = action + ",\n ".join(args) + "\n );\n" action = action + ",\n ".join(args) + "\n );\n"
if 'error' in option: if 'error' in option:
action = action + " PyObject_SetAttrString(py_parser, \"lasterror\", Py_None);\n" action = action + " PyObject_SetAttrString(py_parser, \"lasterror\", Py_None);\n"
action = action + " Py_INCREF(Py_None);\n" action = action + " Py_INCREF(Py_None);\n"
action = action + " yyclearin;\n" action = action + " yyclearin;\n"
action = action + " if (PyObject_HasAttrString($$, \"_pyBisonError\"))\n" action = action + " if (PyObject_HasAttrString($$, \"_pyBisonError\"))\n"
action = action + " {\n" action = action + " {\n"
action = action + " yyerror(PyString_AsString(PyObject_GetAttrString(py_parser, \"lasterror\")));\n" action = action + " yyerror(PyString_AsString(PyObject_GetAttrString(py_parser, \"lasterror\")));\n"
...@@ -434,9 +433,9 @@ cdef class ParserEngine: ...@@ -434,9 +433,9 @@ cdef class ParserEngine:
write(" | ".join(options) + " ;\n\n") write(" | ".join(options) + " ;\n\n")
except: except:
traceback.print_exc() traceback.print_exc()
write("\n\n%%\n\n") write("\n\n%%\n\n")
# now generate C code # now generate C code
epilogue = "\n".join([ epilogue = "\n".join([
'void do_parse(void *parser1,', 'void do_parse(void *parser1,',
...@@ -474,10 +473,10 @@ cdef class ParserEngine: ...@@ -474,10 +473,10 @@ cdef class ParserEngine:
"}", "}",
]) + "\n" ]) + "\n"
write(epilogue) write(epilogue)
# done with grammar file # done with grammar file
f.close() f.close()
# ----------------------------------------------- # -----------------------------------------------
# now generate the lex script # now generate the lex script
if os.path.isfile(buildDirectory + parser.flexFile): if os.path.isfile(buildDirectory + parser.flexFile):
...@@ -547,11 +546,11 @@ cdef class ParserEngine: ...@@ -547,11 +546,11 @@ cdef class ParserEngine:
# compile bison and lex c sources # compile bison and lex c sources
#bisonObj = ccompiler.compile([parser.bisonCFile1]) #bisonObj = ccompiler.compile([parser.bisonCFile1])
#lexObj = ccompiler.compile([parser.flexCFile1]) #lexObj = ccompiler.compile([parser.flexCFile1])
#cl /DWIN32 /G4 /Gs /Oit /MT /nologo /W3 /WX bisondynlib-win32.c /Id:\python23\include #cl /DWIN32 /G4 /Gs /Oit /MT /nologo /W3 /WX bisondynlib-win32.c /Id:\python23\include
#cc.compile(['bisondynlib-win32.c'], #cc.compile(['bisondynlib-win32.c'],
# extra_preargs=['/DWIN32', '/G4', '/Gs', '/Oit', '/MT', '/nologo', '/W3', '/WX', '/Id:\python23\include']) # extra_preargs=['/DWIN32', '/G4', '/Gs', '/Oit', '/MT', '/nologo', '/W3', '/WX', '/Id:\python23\include'])
# link 'em into a shared lib # link 'em into a shared lib
objs = ccompiler.compile([buildDirectory + parser.bisonCFile1, objs = ccompiler.compile([buildDirectory + parser.bisonCFile1,
buildDirectory + parser.flexCFile1], buildDirectory + parser.flexCFile1],
...@@ -571,10 +570,10 @@ cdef class ParserEngine: ...@@ -571,10 +570,10 @@ cdef class ParserEngine:
print 'linking: %s => %s' % (', '.join(objs), libFileName) print 'linking: %s => %s' % (', '.join(objs), libFileName)
ccompiler.link_shared_object(objs, libFileName) ccompiler.link_shared_object(objs, libFileName)
#incdir = PyString_AsString(get_python_inc()) #incdir = PyString_AsString(get_python_inc())
#bisondynlib_build(self.libFilename_py, incdir) #bisondynlib_build(self.libFilename_py, incdir)
# -------------------------------------------- # --------------------------------------------
# clean up, if we succeeded # clean up, if we succeeded
hitlist = objs[:] hitlist = objs[:]
...@@ -598,17 +597,13 @@ cdef class ParserEngine: ...@@ -598,17 +597,13 @@ cdef class ParserEngine:
os.unlink(f) os.unlink(f)
except: except:
print "Warning: failed to delete temporary file %s" % f print "Warning: failed to delete temporary file %s" % f
#@-node:buildLib
#@+node:closeLib
def closeLib(self): def closeLib(self):
""" """
Does the necessary cleanups and closes the parser library Does the necessary cleanups and closes the parser library
""" """
bisondynlib_close(self.libHandle) bisondynlib_close(self.libHandle)
#@-node:closeLib
#@+node:runEngine
def runEngine(self, debug=0): def runEngine(self, debug=0):
""" """
Runs the binary parser engine, as loaded from the lib Runs the binary parser engine, as loaded from the lib
...@@ -624,25 +619,15 @@ cdef class ParserEngine: ...@@ -624,25 +619,15 @@ cdef class ParserEngine:
cbvoid = <void *>py_callback cbvoid = <void *>py_callback
invoid = <void *>py_input invoid = <void *>py_input
if parser.verbose:
print "runEngine: about to call, py_input=0x%lx..." % (<int>invoid)
return bisondynlib_run(handle, parser, cbvoid, invoid, debug) return bisondynlib_run(handle, parser, cbvoid, invoid, debug)
if parser.verbose:
print "runEngine: back from parser"
#@-node:runEngine
#@+node:__del__
def __del__(self): def __del__(self):
""" """
Clean up and bail Clean up and bail
""" """
self.closeLib() self.closeLib()
#@-node:__del__
#@-others
#@-node:cdef class ParserEngine
#@+node:cmpLines
def cmpLines(meth1, meth2): def cmpLines(meth1, meth2):
""" """
Used as a sort() argument for sorting parse target handler methods by Used as a sort() argument for sorting parse target handler methods by
...@@ -654,30 +639,29 @@ def cmpLines(meth1, meth2): ...@@ -654,30 +639,29 @@ def cmpLines(meth1, meth2):
except: except:
line1 = meth1.__init__.func_code.co_firstlineno line1 = meth1.__init__.func_code.co_firstlineno
line2 = meth2.__init__.func_code.co_firstlineno line2 = meth2.__init__.func_code.co_firstlineno
return cmp(line1, line2) return cmp(line1, line2)
#@-node:cmpLines
#@+node:hashParserObject
def hashParserObject(parser): def hashParserObject(parser):
""" """
Calculates an sha1 hex 'hash' of the lex script Calculates an sha1 hex 'hash' of the lex script
and grammar rules in a parser class instance. and grammar rules in a parser class instance.
This is based on the raw text of the lex script attribute, This is based on the raw text of the lex script attribute,
and the grammar rule docstrings within the handler methods. and the grammar rule docstrings within the handler methods.
Used to detect if someone has changed any grammar rules or Used to detect if someone has changed any grammar rules or
lex script, and therefore, whether a shared parser lib rebuild lex script, and therefore, whether a shared parser lib rebuild
is required. is required.
""" """
hasher = sha.new() hasher = sha.new()
# add the lex script # add the lex script
hasher.update(parser.lexscript) hasher.update(parser.lexscript)
# add the tokens # add the tokens
# workaround pyrex weirdness # workaround pyrex weirdness
tokens = list(parser.tokens) tokens = list(parser.tokens)
hasher.update(",".join(list(parser.tokens))) hasher.update(",".join(list(parser.tokens)))
...@@ -685,7 +669,7 @@ def hashParserObject(parser): ...@@ -685,7 +669,7 @@ def hashParserObject(parser):
# add the precedences # add the precedences
for direction, tokens in parser.precedences: for direction, tokens in parser.precedences:
hasher.update(direction + "".join(tokens)) hasher.update(direction + "".join(tokens))
# extract the parser target handler names # extract the parser target handler names
handlerNames = dir(parser) handlerNames = dir(parser)
...@@ -706,18 +690,11 @@ def hashParserObject(parser): ...@@ -706,18 +690,11 @@ def hashParserObject(parser):
if callable(attr): if callable(attr):
tmp.append(attr) tmp.append(attr)
handlers = tmp handlers = tmp
# now add in the methods' docstrings # now add in the methods' docstrings
for h in handlers: for h in handlers:
docString = h.__doc__ docString = h.__doc__
hasher.update(docString) hasher.update(docString)
# done # done
return hasher.hexdigest() return hasher.hexdigest()
#@-node:hashParserObject
#@-others
#@-node:@file src/pyrex/bison_.pyx
#@-leo
...@@ -52,13 +52,13 @@ class BisonError: ...@@ -52,13 +52,13 @@ class BisonError:
class BisonNode: class BisonNode:
""" """
Generic class for wrapping parse targets. Generic class for wrapping parse targets.
Arguments: Arguments:
- targetname - the name of the parse target being wrapped. - targetname - the name of the parse target being wrapped.
- items - optional - a list of items comprising a clause - items - optional - a list of items comprising a clause
in the target rule - typically this will only be used in the target rule - typically this will only be used
by the PyBison callback mechanism. by the PyBison callback mechanism.
Keywords: Keywords:
- any keywords you want (except 'items'), with any type of value. - any keywords you want (except 'items'), with any type of value.
keywords will be stored as attributes in the constructed object. keywords will be stored as attributes in the constructed object.
...@@ -66,37 +66,37 @@ class BisonNode: ...@@ -66,37 +66,37 @@ class BisonNode:
#@ @+others #@ @+others
#@+node:__init__ #@+node:__init__
def __init__(self, **kw): def __init__(self, **kw):
self.__dict__.update(kw) self.__dict__.update(kw)
# ensure some default attribs # ensure some default attribs
self.target = kw.get('target', 'UnnamedTarget') self.target = kw.get('target', 'UnnamedTarget')
self.names = kw.get('names', []) self.names = kw.get('names', [])
self.values = kw.get('values', []) self.values = kw.get('values', [])
self.option = kw.get('option', 0) self.option = kw.get('option', 0)
# mirror this dict to simplify dumping # mirror this dict to simplify dumping
self.kw = kw self.kw = kw
#@-node:__init__ #@-node:__init__
#@+node:__str__ #@+node:__str__
def __str__(self): def __str__(self):
return "<BisonNode:%s>" % self.target return "<BisonNode:%s>" % self.target
#@-node:__str__ #@-node:__str__
#@+node:__repr__ #@+node:__repr__
def __repr__(self): def __repr__(self):
return str(self) return str(self)
#@-node:__repr__ #@-node:__repr__
#@+node:__getitem__ #@+node:__getitem__
def __getitem__(self, item): def __getitem__(self, item):
""" """
Retrieves the ith value from this node, or child nodes Retrieves the ith value from this node, or child nodes
If the subscript is a single number, it will be used as an If the subscript is a single number, it will be used as an
index into this node's children list. index into this node's children list.
If the subscript is a list or tuple, we recursively fetch If the subscript is a list or tuple, we recursively fetch
the item by using the first element as an index into this the item by using the first element as an index into this
node's children, the second element as an index into that node's children, the second element as an index into that
...@@ -113,7 +113,7 @@ class BisonNode: ...@@ -113,7 +113,7 @@ class BisonNode:
#@-node:__getitem__ #@-node:__getitem__
#@+node:__len__ #@+node:__len__
def __len__(self): def __len__(self):
return len(self.values) return len(self.values)
#@-node:__len__ #@-node:__len__
#@+node:__getslice__ #@+node:__getslice__
...@@ -133,7 +133,7 @@ class BisonNode: ...@@ -133,7 +133,7 @@ class BisonNode:
indents = " " * indent * 2 indents = " " * indent * 2
#print "%s%s: %s %s" % (indents, self.target, self.option, self.names) #print "%s%s: %s %s" % (indents, self.target, self.option, self.names)
print "%s%s:" % (indents, self.target) print "%s%s:" % (indents, self.target)
for name, val in self.kw.items() + zip(self.names, self.values): for name, val in self.kw.items() + zip(self.names, self.values):
if name in specialAttribs or name.startswith("_"): if name in specialAttribs or name.startswith("_"):
continue continue
...@@ -141,18 +141,18 @@ class BisonNode: ...@@ -141,18 +141,18 @@ class BisonNode:
val.dump(indent+1) val.dump(indent+1)
else: else:
print indents + " %s=%s" % (name, val) print indents + " %s=%s" % (name, val)
#@-node:dump #@-node:dump
#@+node:toxml #@+node:toxml
def toxml(self): def toxml(self):
""" """
Returns an xml serialisation of this node and its children, as a raw string Returns an xml serialisation of this node and its children, as a raw string
Called on the toplevel node, the xml is a representation of the Called on the toplevel node, the xml is a representation of the
entire parse tree. entire parse tree.
""" """
return self.toxmldoc().toxml() return self.toxmldoc().toxml()
#@-node:toxml #@-node:toxml
#@+node:toprettyxml #@+node:toprettyxml
def toprettyxml(self, indent=' ', newl='\n', encoding=None): def toprettyxml(self, indent=' ', newl='\n', encoding=None):
...@@ -162,7 +162,7 @@ class BisonNode: ...@@ -162,7 +162,7 @@ class BisonNode:
return self.toxmldoc().toprettyxml(indent=indent, return self.toxmldoc().toprettyxml(indent=indent,
newl=newl, newl=newl,
encoding=encoding) encoding=encoding)
#@-node:toprettyxml #@-node:toprettyxml
#@+node:toxmldoc #@+node:toxmldoc
def toxmldoc(self): def toxmldoc(self):
...@@ -172,7 +172,7 @@ class BisonNode: ...@@ -172,7 +172,7 @@ class BisonNode:
d = xml.dom.minidom.Document() d = xml.dom.minidom.Document()
d.appendChild(self.toxmlelem(d)) d.appendChild(self.toxmlelem(d))
return d return d
#@-node:toxmldoc #@-node:toxmldoc
#@+node:toxmlelem #@+node:toxmlelem
def toxmlelem(self, docobj): def toxmlelem(self, docobj):
...@@ -180,10 +180,10 @@ class BisonNode: ...@@ -180,10 +180,10 @@ class BisonNode:
Returns a DOM Element object of this node and its children Returns a DOM Element object of this node and its children
""" """
specialAttribs = ['option', 'target', 'names', 'values'] specialAttribs = ['option', 'target', 'names', 'values']
# generate an xml element obj for this node # generate an xml element obj for this node
x = docobj.createElement(self.target) x = docobj.createElement(self.target)
# set attribs # set attribs
for name, val in self.kw.items(): for name, val in self.kw.items():
if name in ['names', 'values'] or name.startswith("_"): if name in ['names', 'values'] or name.startswith("_"):
...@@ -191,7 +191,7 @@ class BisonNode: ...@@ -191,7 +191,7 @@ class BisonNode:
x.setAttribute(name, str(val)) x.setAttribute(name, str(val))
#x.setAttribute('target', self.target) #x.setAttribute('target', self.target)
#x.setAttribute('option', self.option) #x.setAttribute('option', self.option)
# and add the children # and add the children
for name, val in zip(self.names, self.values): for name, val in zip(self.names, self.values):
if name in specialAttribs or name.startswith("_"): if name in specialAttribs or name.startswith("_"):
...@@ -204,11 +204,11 @@ class BisonNode: ...@@ -204,11 +204,11 @@ class BisonNode:
tn = docobj.createTextNode(val) tn = docobj.createTextNode(val)
sn.appendChild(tn) sn.appendChild(tn)
x.appendChild(sn) x.appendChild(sn)
# done # done
return x return x
#@-node:toxmlelem #@-node:toxmlelem
#@-others #@-others
#@-node:class BisonNode #@-node:class BisonNode
...@@ -216,7 +216,7 @@ class BisonNode: ...@@ -216,7 +216,7 @@ class BisonNode:
class BisonParser(object): class BisonParser(object):
""" """
Base parser class Base parser class
You should subclass this, and provide a bunch of methods called You should subclass this, and provide a bunch of methods called
'on_TargetName', where 'TargetName' is the name of each target in 'on_TargetName', where 'TargetName' is the name of each target in
your grammar (.y) file. your grammar (.y) file.
...@@ -225,21 +225,21 @@ class BisonParser(object): ...@@ -225,21 +225,21 @@ class BisonParser(object):
#@+node:attributes #@+node:attributes
# --------------------------------------- # ---------------------------------------
# override these if you need to # override these if you need to
# command and options for running yacc/bison, except for filename arg # command and options for running yacc/bison, except for filename arg
bisonCmd = ["bison", "-d", "-v", '-t'] bisonCmd = ["bison", "-d", "-v", '-t']
bisonFile = "tmp.y" bisonFile = "tmp.y"
bisonCFile = "tmp.tab.c" bisonCFile = "tmp.tab.c"
bisonHFile = "tmp.tab.h" # name of header file generated by bison cmd bisonHFile = "tmp.tab.h" # name of header file generated by bison cmd
bisonCFile1 = "tmp.bison.c" # c output file from bison gets renamed to this bisonCFile1 = "tmp.bison.c" # c output file from bison gets renamed to this
bisonHFile1 = "tokens.h" # bison-generated header file gets renamed to this bisonHFile1 = "tokens.h" # bison-generated header file gets renamed to this
flexCmd = ["flex", ] # command and options for running [f]lex, except for filename arg flexCmd = ["flex", ] # command and options for running [f]lex, except for filename arg
flexFile = "tmp.l" flexFile = "tmp.l"
flexCFile = "lex.yy.c" flexCFile = "lex.yy.c"
flexCFile1 = "tmp.lex.c" # c output file from lex gets renamed to this flexCFile1 = "tmp.lex.c" # c output file from lex gets renamed to this
cflags_pre = ['-fPIC'] # = CFLAGS added before all arguments. cflags_pre = ['-fPIC'] # = CFLAGS added before all arguments.
...@@ -247,27 +247,27 @@ class BisonParser(object): ...@@ -247,27 +247,27 @@ class BisonParser(object):
buildDirectory = './' # Directory used to store the generated / compiled files. buildDirectory = './' # Directory used to store the generated / compiled files.
debugSymbols = 1 # Add debugging symbols to the binary files. debugSymbols = 1 # Add debugging symbols to the binary files.
verbose = 0 verbose = 0
file = None # default to sys.stdin file = None # default to sys.stdin
last = None # last parsed target, top of parse tree last = None # last parsed target, top of parse tree
lasterror = None # gets set if there was an error lasterror = None # gets set if there was an error
keepfiles = 0 # set to 1 to keep temporary engine build files keepfiles = 0 # set to 1 to keep temporary engine build files
bisonEngineLibName = None # defaults to 'modulename-engine' bisonEngineLibName = None # defaults to 'modulename-engine'
defaultNodeClass = BisonNode # class to use by default for creating new parse nodes defaultNodeClass = BisonNode # class to use by default for creating new parse nodes
#@-node:attributes #@-node:attributes
#@+node:__init__ #@+node:__init__
def __init__(self, **kw): def __init__(self, **kw):
""" """
Abstract representation of parser Abstract representation of parser
Keyword arguments: Keyword arguments:
- read - a callable accepting an int arg (nbytes) and returning a string, - read - a callable accepting an int arg (nbytes) and returning a string,
default is this class' read() method default is this class' read() method
...@@ -285,7 +285,7 @@ class BisonParser(object): ...@@ -285,7 +285,7 @@ class BisonParser(object):
read = kw.get('read', None) read = kw.get('read', None)
if read: if read:
self.read = read self.read = read
fileobj = kw.get('file', None) fileobj = kw.get('file', None)
if fileobj: if fileobj:
if type(fileobj) == type(""): if type(fileobj) == type(""):
...@@ -296,34 +296,31 @@ class BisonParser(object): ...@@ -296,34 +296,31 @@ class BisonParser(object):
self.file = fileobj self.file = fileobj
else: else:
self.file = sys.stdin self.file = sys.stdin
nodeClass = kw.get('defaultNodeClass', None) nodeClass = kw.get('defaultNodeClass', None)
if nodeClass: if nodeClass:
self.defaultNodeClass = nodeClass self.defaultNodeClass = nodeClass
self.verbose = kw.get('verbose', 0) self.verbose = kw.get('verbose', 0)
if kw.has_key('keepfiles'): if kw.has_key('keepfiles'):
self.keepfiles = kw['keepfiles'] self.keepfiles = kw['keepfiles']
# if engine lib name not declared, invent ont # if engine lib name not declared, invent ont
if not self.bisonEngineLibName: if not self.bisonEngineLibName:
self.bisonEngineLibName = self.__class__.__module__ + "-parser" self.bisonEngineLibName = self.__class__.__module__ + "-parser"
# get an engine # get an engine
self.engine = ParserEngine(self) self.engine = ParserEngine(self)
#@-node:__init__
#@+node:__getattr__
def __getitem__(self, idx): def __getitem__(self, idx):
return self.last[idx] return self.last[idx]
#@-node:__getattr__
#@+node:_handle
def _handle(self, targetname, option, names, values): def _handle(self, targetname, option, names, values):
""" """
Callback which receives a target from parser, as a targetname Callback which receives a target from parser, as a targetname
and list of term names and values. and list of term names and values.
Tries to dispatch to on_TargetName() methods if they exist, Tries to dispatch to on_TargetName() methods if they exist,
otherwise wraps the target in a BisonNode object otherwise wraps the target in a BisonNode object
""" """
...@@ -347,20 +344,17 @@ class BisonParser(object): ...@@ -347,20 +344,17 @@ class BisonParser(object):
if self.verbose: if self.verbose:
print "no handler for %s, using default" % targetname print "no handler for %s, using default" % targetname
self.last = BisonNode(targetname, option=option, names=names, values=values) self.last = BisonNode(targetname, option=option, names=names, values=values)
# reset any resulting errors (assume they've been handled) # reset any resulting errors (assume they've been handled)
#self.lasterror = None #self.lasterror = None
# assumedly the last thing parsed is at the top of the tree # assumedly the last thing parsed is at the top of the tree
return self.last return self.last
#@-node:_handle
#@+node:run
def run(self, **kw): def run(self, **kw):
""" """
Runs the parser, and returns the top-most parse target. Runs the parser, and returns the top-most parse target.
Keywords: Keywords:
- file - either a string, comprising a file to open and read input from, or - file - either a string, comprising a file to open and read input from, or
a Python file object a Python file object
...@@ -368,7 +362,7 @@ class BisonParser(object): ...@@ -368,7 +362,7 @@ class BisonParser(object):
""" """
if self.verbose: if self.verbose:
print "Parser.run: calling engine" print "Parser.run: calling engine"
# grab keywords # grab keywords
fileobj = kw.get('file', self.file) fileobj = kw.get('file', self.file)
if type(fileobj) == type(""): if type(fileobj) == type(""):
...@@ -380,47 +374,44 @@ class BisonParser(object): ...@@ -380,47 +374,44 @@ class BisonParser(object):
else: else:
filename = None filename = None
fileobj = None fileobj = None
read = kw.get('read', self.read) read = kw.get('read', self.read)
debug = kw.get('debug', 0) debug = kw.get('debug', 0)
# back up existing attribs # back up existing attribs
oldfile = self.file oldfile = self.file
oldread = self.read oldread = self.read
# plug in new ones, if given # plug in new ones, if given
if fileobj: if fileobj:
self.file = fileobj self.file = fileobj
if read: if read:
self.read = read self.read = read
# do the parsing job, spew if error # do the parsing job, spew if error
self.lasterror = None self.lasterror = None
self.engine.runEngine(debug) self.engine.runEngine(debug)
if self.lasterror: if self.lasterror:
#print "Got error: %s" % repr(self.error) #print "Got error: %s" % repr(self.error)
if filename != None: if filename != None:
raise ParserSyntaxError("%s:%d: '%s' near '%s'" % ((filename,) + self.lasterror)) raise ParserSyntaxError("%s:%d: '%s' near '%s'" % ((filename,) + self.lasterror))
else: else:
raise ParserSyntaxError("Line %d: '%s' near '%s'" % self.lasterror) raise ParserSyntaxError("Line %d: '%s' near '%s'" % self.lasterror)
# restore old values # restore old values
self.file = oldfile self.file = oldfile
self.read = oldread self.read = oldread
if self.verbose: if self.verbose:
print "Parser.run: back from engine" print "Parser.run: back from engine"
return self.last return self.last
#@-node:run
#@+node:read
def read(self, nbytes): def read(self, nbytes):
""" """
Override this in your subclass, if you desire. Override this in your subclass, if you desire.
Arguments: Arguments:
- nbytes - the maximum length of the string which you may return. - nbytes - the maximum length of the string which you may return.
DO NOT return a string longer than this, or else Bad Things will DO NOT return a string longer than this, or else Bad Things will
...@@ -433,56 +424,48 @@ class BisonParser(object): ...@@ -433,56 +424,48 @@ class BisonParser(object):
if self.verbose: if self.verbose:
print "Parser.read: got %s bytes" % len(bytes) print "Parser.read: got %s bytes" % len(bytes)
return bytes return bytes
#@-node:read
#@+node:_error
def _error(self, linenum, msg, tok): def _error(self, linenum, msg, tok):
print "Parser: line %s: syntax error '%s' before '%s'" % (linenum, msg, tok) print "Parser: line %s: syntax error '%s' before '%s'" % (linenum, msg, tok)
#@-node:_error
#@+node:error
def error(self, value): def error(self, value):
""" """
Return the result of this method from a handler to notify a syntax error Return the result of this method from a handler to notify a syntax error
""" """
self.lasterror = value self.lasterror = value
return BisonError(value) return BisonError(value)
#@-node:error
#@+node:toxml
def toxml(self): def toxml(self):
""" """
Serialises the parse tree and returns it as a raw xml string Serialises the parse tree and returns it as a raw xml string
""" """
return self.last.toxml() return self.last.toxml()
#@-node:toxml
#@+node:toxmldoc
def toxmldoc(self): def toxmldoc(self):
""" """
Returns an xml.dom.minidom.Document object containing the parse tree Returns an xml.dom.minidom.Document object containing the parse tree
""" """
return self.last.toxmldoc() return self.last.toxmldoc()
#@-node:toxmldoc
#@+node:toprettyxml
def toprettyxml(self): def toprettyxml(self):
""" """
Returns a human-readable xml representation of the parse tree Returns a human-readable xml representation of the parse tree
""" """
return self.last.toprettyxml() return self.last.toprettyxml()
#@-node:toprettyxml
#@+node:loadxml
def loadxml(self, raw, namespace=None): def loadxml(self, raw, namespace=None):
""" """
Loads a parse tree from raw xml text Loads a parse tree from raw xml text
Stores it in the '.last' attribute, which is where the root node Stores it in the '.last' attribute, which is where the root node
of parsed text gets stored of parsed text gets stored
Arguments: Arguments:
- raw - string containing the raw xml - raw - string containing the raw xml
- namespace - a dict or module object, where the node classes required for - namespace - a dict or module object, where the node classes required for
reconstituting the parse tree, can be found reconstituting the parse tree, can be found
Returns: Returns:
- root node object of reconstituted parse tree - root node object of reconstituted parse tree
""" """
...@@ -490,26 +473,24 @@ class BisonParser(object): ...@@ -490,26 +473,24 @@ class BisonParser(object):
tree = self.loadxmldoc(doc, namespace) tree = self.loadxmldoc(doc, namespace)
self.last = tree self.last = tree
return tree return tree
#@-node:loadxml
#@+node:loadxmldoc
def loadxmldoc(self, xmldoc, namespace=None): def loadxmldoc(self, xmldoc, namespace=None):
""" """
Returns a reconstituted parse tree, loaded from an Returns a reconstituted parse tree, loaded from an
xml.dom.minidom.Document instance xml.dom.minidom.Document instance
Arguments: Arguments:
- xmldoc - an xml.dom.minidom.Document instance - xmldoc - an xml.dom.minidom.Document instance
- namespace - a dict from which to find the classes needed - namespace - a dict from which to find the classes needed
to translate the document into a tree of parse nodes to translate the document into a tree of parse nodes
""" """
return self.loadxmlobj(xmldoc.childNodes[0], namespace) return self.loadxmlobj(xmldoc.childNodes[0], namespace)
#@-node:loadxmldoc
#@+node:loadxmlobj
def loadxmlobj(self, xmlobj, namespace=None): def loadxmlobj(self, xmlobj, namespace=None):
""" """
Returns a node object, being a parse tree, reconstituted from an Returns a node object, being a parse tree, reconstituted from an
xml.dom.minidom.Element object xml.dom.minidom.Element object
Arguments: Arguments:
- xmlobj - an xml.dom.minidom.Element instance - xmlobj - an xml.dom.minidom.Element instance
- namespace - a namespace from which the node classes - namespace - a namespace from which the node classes
...@@ -520,34 +501,34 @@ class BisonParser(object): ...@@ -520,34 +501,34 @@ class BisonParser(object):
namespace = namespace.__dict__ namespace = namespace.__dict__
elif namespace == None: elif namespace == None:
namespace = globals() namespace = globals()
objname = xmlobj.tagName objname = xmlobj.tagName
classname = objname + "_Node" classname = objname + "_Node"
classobj = namespace.get(classname, None) classobj = namespace.get(classname, None)
namespacekeys = namespace.keys() namespacekeys = namespace.keys()
# barf if node is not a known parse node or token # barf if node is not a known parse node or token
if (not classobj) and objname not in self.tokens: if (not classobj) and objname not in self.tokens:
raise Exception("Cannot reconstitute %s: can't find required node class or token %s" % ( raise Exception("Cannot reconstitute %s: can't find required node class or token %s" % (
objname, classname)) objname, classname))
if classobj: if classobj:
nodeobj = classobj() nodeobj = classobj()
# add the attribs # add the attribs
for k,v in xmlobj.attributes.items(): for k,v in xmlobj.attributes.items():
setattr(nodeobj, k, v) setattr(nodeobj, k, v)
else: else:
nodeobj = None nodeobj = None
#print "----------------" #print "----------------"
#print "objname=%s" % repr(objname) #print "objname=%s" % repr(objname)
#print "classname=%s" % repr(classname) #print "classname=%s" % repr(classname)
#print "classobj=%s" % repr(classobj) #print "classobj=%s" % repr(classobj)
#print "nodeobj=%s" % repr(nodeobj) #print "nodeobj=%s" % repr(nodeobj)
# now add the children # now add the children
for child in xmlobj.childNodes: for child in xmlobj.childNodes:
#print "%s attributes=%s" % (child, child.attributes.items()) #print "%s attributes=%s" % (child, child.attributes.items())
...@@ -560,22 +541,15 @@ class BisonParser(object): ...@@ -560,22 +541,15 @@ class BisonParser(object):
# it's a token # it's a token
childobj = child.childNodes[0].nodeValue childobj = child.childNodes[0].nodeValue
#print "got token %s=%s" % (childname, childobj) #print "got token %s=%s" % (childname, childobj)
nodeobj.names.append(childname) nodeobj.names.append(childname)
nodeobj.values.append(childobj) nodeobj.values.append(childobj)
# done
return nodeobj return nodeobj
#@-node:loadxmlobj
#@+node:_globals
def _globals(self): def _globals(self):
return globals().keys() return globals().keys()
#@-node:_globals
#@-others
#@-node:class BisonParser
#@+node:bisonToPython
def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0): def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
""" """
Rips the rules, tokens and precedences from a bison file, and the Rips the rules, tokens and precedences from a bison file, and the
...@@ -610,7 +584,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0): ...@@ -610,7 +584,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
except: except:
raise Exception("Cannot open lex file %s" % lexfileName) raise Exception("Cannot open lex file %s" % lexfileName)
# break up into the three '%%'-separated sections # break up into the three '%%'-separated sections
try: try:
prologue, rulesRaw, epilogue = rawBison.split("\n%%\n") prologue, rulesRaw, epilogue = rawBison.split("\n%%\n")
except: except:
...@@ -618,13 +592,13 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0): ...@@ -618,13 +592,13 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
"File %s is not a properly formatted bison file" "File %s is not a properly formatted bison file"
" (needs 3 sections separated by %%%%" % (bisonfileName) " (needs 3 sections separated by %%%%" % (bisonfileName)
) )
# -------------------------------------- # --------------------------------------
# process prologue # process prologue
prologue = prologue.split("%}")[-1].strip() # ditch the C code prologue = prologue.split("%}")[-1].strip() # ditch the C code
prologue = re.sub("\\n([\t ]+)", " ", prologue) # join broken lines prologue = re.sub("\\n([\t ]+)", " ", prologue) # join broken lines
#prologueLines = [line.strip() for line in prologue.split("\n")] #prologueLines = [line.strip() for line in prologue.split("\n")]
lines = prologue.split("\n") lines = prologue.split("\n")
tmp = [] tmp = []
...@@ -685,7 +659,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0): ...@@ -685,7 +659,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
terms = tmp terms = tmp
rules.append((tgt, terms)) rules.append((tgt, terms))
# now we have our rulebase, we can churn out our skeleton Python file # now we have our rulebase, we can churn out our skeleton Python file
pyfile.write("\n".join([ pyfile.write("\n".join([
'#!/usr/bin/env python', '#!/usr/bin/env python',
...@@ -737,7 +711,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0): ...@@ -737,7 +711,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
'# ------------------------------------------------------', '# ------------------------------------------------------',
'\n', '\n',
])) ]))
# now spit out class decs for every parse target # now spit out class decs for every parse target
for target, options in rules: for target, options in rules:
tmp = [] tmp = []
...@@ -763,7 +737,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0): ...@@ -763,7 +737,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
'\n', '\n',
])) ]))
# start churning out the class dec # start churning out the class dec
pyfile.write("\n".join([ pyfile.write("\n".join([
'class Parser(BisonParser):', 'class Parser(BisonParser):',
...@@ -942,8 +916,3 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0): ...@@ -942,8 +916,3 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
'', '',
'', '',
])) ]))
#@-node:bisonToPython
#@-others
#@-node:@file src/python/bison.py
#@-leo
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment