Skip to content

P
pybison
Commit df0ac6f3 authored by Sander Mathijs van Veen's avatar Sander Mathijs van Veen
Browse files
Options

Code cleanup.

parent f9a34e5e
Hide whitespace changes
Inline Side-by-side
Showing with 137 additions and 191 deletions
src/pyrex/bison_.pyx
View file @ df0ac6f3
......@@ -54,6 +54,7 @@ cdef extern from "../c/bisondynlib.h":
#int bisondynlib_build(char *libName, char *includedir)
# Definitions for variadic functions (e.g. py_callback).
cdef extern from "stdarg.h":
ctypedef struct va_list:
pass
......@@ -163,7 +164,7 @@ cdef class ParserEngine:
- calling the entry point
- closing the library
Makes direct calls to the platform-dependent routines in
Makes direct calls to the platform-dependent routines in
bisondynlib-[linux|windows].c
"""
cdef object parser
......@@ -246,12 +247,12 @@ cdef class ParserEngine:
cdef char *libFilename
cdef char *err
cdef void *handle
# convert python filename string to c string
libFilename = PyString_AsString(self.libFilename_py)
parser = self.parser
if parser.verbose:
print "Opening library %s" % self.libFilename_py
handle = bisondynlib_open(libFilename)
......@@ -260,19 +261,17 @@ cdef class ParserEngine:
if err:
printf("ParserEngine.openLib: error '%s'\n", err)
return
# extract symbols
self.libHash = bisondynlib_lookup_hash(handle)
if parser.verbose:
print "Successfully loaded library"
#@-node:openLib
#@+node:buildLib
def buildLib(self):
"""
Creates the parser engine lib
This consists of:
1. Ripping the tokens list, precedences, start target, handler docstrings
and lex script from this Parser instance's attribs and methods
......@@ -281,11 +280,11 @@ cdef class ParserEngine:
4. Compiling the C files, and link into a dynamic lib
"""
cdef char *incdir
# -------------------------------------------------
# rip the pertinent grammar specs from parser class
parser = self.parser
# get target handler methods, in the order of appearance in the source
# file.
attribs = dir(parser)
......@@ -295,7 +294,7 @@ cdef class ParserEngine:
method = getattr(parser, a)
gHandlers.append(method)
gHandlers.sort(cmpLines)
# get start symbol, tokens, precedences, lex script
gStart = parser.start
gTokens = parser.tokens
......@@ -315,7 +314,7 @@ cdef class ParserEngine:
f = open(buildDirectory + parser.bisonFile, "w")
write = f.write
writelines = f.writelines
# grammar file prologue
write("\n".join([
"%{",
......@@ -366,7 +365,7 @@ cdef class ParserEngine:
options = options.strip()
tmp = []
#print "options = %s" % repr(options)
opts = options.split("|")
#print "opts = %s" % repr(opts)
......@@ -374,15 +373,15 @@ cdef class ParserEngine:
#print "r = <%s>" % r
opts1 = re.split(r, " " + options)
#print "opts1 = %s" % repr(opts1)
for o in opts1:
o = o.strip()
tmp.append(reSpaces.split(o))
options = tmp
rules.append((target, options))
# and render rules to grammar file
for rule in rules:
try:
......@@ -409,19 +408,19 @@ cdef class ParserEngine:
i = i - 1
break # hack for rules using '%prec'
args.append('"%s", $%d' % (option[i], i+1))
# now, we have the correct terms count
action = action % (i + 1)
# assemble the full rule + action, ad to list
action = action + ",\n "
action = action + ",\n ".join(args) + "\n );\n"
if 'error' in option:
action = action + " PyObject_SetAttrString(py_parser, \"lasterror\", Py_None);\n"
action = action + " Py_INCREF(Py_None);\n"
action = action + " yyclearin;\n"
action = action + " if (PyObject_HasAttrString($$, \"_pyBisonError\"))\n"
action = action + " {\n"
action = action + " yyerror(PyString_AsString(PyObject_GetAttrString(py_parser, \"lasterror\")));\n"
......@@ -434,9 +433,9 @@ cdef class ParserEngine:
write(" | ".join(options) + " ;\n\n")
except:
traceback.print_exc()
write("\n\n%%\n\n")
# now generate C code
epilogue = "\n".join([
'void do_parse(void *parser1,',
......@@ -474,10 +473,10 @@ cdef class ParserEngine:
"}",
]) + "\n"
write(epilogue)
# done with grammar file
f.close()
# -----------------------------------------------
# now generate the lex script
if os.path.isfile(buildDirectory + parser.flexFile):
......@@ -547,11 +546,11 @@ cdef class ParserEngine:
# compile bison and lex c sources
#bisonObj = ccompiler.compile([parser.bisonCFile1])
#lexObj = ccompiler.compile([parser.flexCFile1])
#cl /DWIN32 /G4 /Gs /Oit /MT /nologo /W3 /WX bisondynlib-win32.c /Id:\python23\include
#cc.compile(['bisondynlib-win32.c'],
#cc.compile(['bisondynlib-win32.c'],
# extra_preargs=['/DWIN32', '/G4', '/Gs', '/Oit', '/MT', '/nologo', '/W3', '/WX', '/Id:\python23\include'])
# link 'em into a shared lib
objs = ccompiler.compile([buildDirectory + parser.bisonCFile1,
buildDirectory + parser.flexCFile1],
......@@ -571,10 +570,10 @@ cdef class ParserEngine:
print 'linking: %s => %s' % (', '.join(objs), libFileName)
ccompiler.link_shared_object(objs, libFileName)
#incdir = PyString_AsString(get_python_inc())
#bisondynlib_build(self.libFilename_py, incdir)
# --------------------------------------------
# clean up, if we succeeded
hitlist = objs[:]
......@@ -598,17 +597,13 @@ cdef class ParserEngine:
os.unlink(f)
except:
print "Warning: failed to delete temporary file %s" % f
#@-node:buildLib
#@+node:closeLib
def closeLib(self):
"""
Does the necessary cleanups and closes the parser library
"""
bisondynlib_close(self.libHandle)
#@-node:closeLib
#@+node:runEngine
def runEngine(self, debug=0):
"""
Runs the binary parser engine, as loaded from the lib
......@@ -624,25 +619,15 @@ cdef class ParserEngine:
cbvoid = <void *>py_callback
invoid = <void *>py_input
if parser.verbose:
print "runEngine: about to call, py_input=0x%lx..." % (<int>invoid)
return bisondynlib_run(handle, parser, cbvoid, invoid, debug)
if parser.verbose:
print "runEngine: back from parser"
#@-node:runEngine
#@+node:__del__
def __del__(self):
"""
Clean up and bail
"""
self.closeLib()
#@-node:__del__
#@-others
#@-node:cdef class ParserEngine
#@+node:cmpLines
def cmpLines(meth1, meth2):
"""
Used as a sort() argument for sorting parse target handler methods by
......@@ -654,30 +639,29 @@ def cmpLines(meth1, meth2):
except:
line1 = meth1.__init__.func_code.co_firstlineno
line2 = meth2.__init__.func_code.co_firstlineno
return cmp(line1, line2)
#@-node:cmpLines
#@+node:hashParserObject
def hashParserObject(parser):
"""
Calculates an sha1 hex 'hash' of the lex script
and grammar rules in a parser class instance.
This is based on the raw text of the lex script attribute,
and the grammar rule docstrings within the handler methods.
Used to detect if someone has changed any grammar rules or
lex script, and therefore, whether a shared parser lib rebuild
is required.
"""
hasher = sha.new()
# add the lex script
hasher.update(parser.lexscript)
# add the tokens
# workaround pyrex weirdness
tokens = list(parser.tokens)
hasher.update(",".join(list(parser.tokens)))
......@@ -685,7 +669,7 @@ def hashParserObject(parser):
# add the precedences
for direction, tokens in parser.precedences:
hasher.update(direction + "".join(tokens))
# extract the parser target handler names
handlerNames = dir(parser)
......@@ -706,18 +690,11 @@ def hashParserObject(parser):
if callable(attr):
tmp.append(attr)
handlers = tmp
# now add in the methods' docstrings
for h in handlers:
docString = h.__doc__
hasher.update(docString)
# done
return hasher.hexdigest()
#@-node:hashParserObject
#@-others
#@-node:@file src/pyrex/bison_.pyx
#@-leo
src/python/bison.py
View file @ df0ac6f3
......@@ -52,13 +52,13 @@ class BisonError:
class BisonNode:
"""
Generic class for wrapping parse targets.
Arguments:
- targetname - the name of the parse target being wrapped.
- items - optional - a list of items comprising a clause
in the target rule - typically this will only be used
by the PyBison callback mechanism.
Keywords:
- any keywords you want (except 'items'), with any type of value.
keywords will be stored as attributes in the constructed object.
......@@ -66,37 +66,37 @@ class BisonNode:
#@ @+others
#@+node:__init__
def __init__(self, **kw):
self.__dict__.update(kw)
# ensure some default attribs
self.target = kw.get('target', 'UnnamedTarget')
self.names = kw.get('names', [])
self.values = kw.get('values', [])
self.option = kw.get('option', 0)
# mirror this dict to simplify dumping
self.kw = kw
#@-node:__init__
#@+node:__str__
def __str__(self):
return "<BisonNode:%s>" % self.target
#@-node:__str__
#@+node:__repr__
def __repr__(self):
return str(self)
#@-node:__repr__
#@+node:__getitem__
def __getitem__(self, item):
"""
Retrieves the ith value from this node, or child nodes
If the subscript is a single number, it will be used as an
index into this node's children list.
If the subscript is a list or tuple, we recursively fetch
the item by using the first element as an index into this
node's children, the second element as an index into that
......@@ -113,7 +113,7 @@ class BisonNode:
#@-node:__getitem__
#@+node:__len__
def __len__(self):
return len(self.values)
#@-node:__len__
#@+node:__getslice__
......@@ -133,7 +133,7 @@ class BisonNode:
indents = " " * indent * 2
#print "%s%s: %s %s" % (indents, self.target, self.option, self.names)
print "%s%s:" % (indents, self.target)
for name, val in self.kw.items() + zip(self.names, self.values):
if name in specialAttribs or name.startswith("_"):
continue
......@@ -141,18 +141,18 @@ class BisonNode:
val.dump(indent+1)
else:
print indents + " %s=%s" % (name, val)
#@-node:dump
#@+node:toxml
def toxml(self):
"""
Returns an xml serialisation of this node and its children, as a raw string
Called on the toplevel node, the xml is a representation of the
entire parse tree.
"""
return self.toxmldoc().toxml()
#@-node:toxml
#@+node:toprettyxml
def toprettyxml(self, indent=' ', newl='\n', encoding=None):
......@@ -162,7 +162,7 @@ class BisonNode:
return self.toxmldoc().toprettyxml(indent=indent,
newl=newl,
encoding=encoding)
#@-node:toprettyxml
#@+node:toxmldoc
def toxmldoc(self):
......@@ -172,7 +172,7 @@ class BisonNode:
d = xml.dom.minidom.Document()
d.appendChild(self.toxmlelem(d))
return d
#@-node:toxmldoc
#@+node:toxmlelem
def toxmlelem(self, docobj):
......@@ -180,10 +180,10 @@ class BisonNode:
Returns a DOM Element object of this node and its children
"""
specialAttribs = ['option', 'target', 'names', 'values']
# generate an xml element obj for this node
x = docobj.createElement(self.target)
# set attribs
for name, val in self.kw.items():
if name in ['names', 'values'] or name.startswith("_"):
......@@ -191,7 +191,7 @@ class BisonNode:
x.setAttribute(name, str(val))
#x.setAttribute('target', self.target)
#x.setAttribute('option', self.option)
# and add the children
for name, val in zip(self.names, self.values):
if name in specialAttribs or name.startswith("_"):
......@@ -204,11 +204,11 @@ class BisonNode:
tn = docobj.createTextNode(val)
sn.appendChild(tn)
x.appendChild(sn)
# done
return x
#@-node:toxmlelem
#@-others
#@-node:class BisonNode
......@@ -216,7 +216,7 @@ class BisonNode:
class BisonParser(object):
"""
Base parser class
You should subclass this, and provide a bunch of methods called
'on_TargetName', where 'TargetName' is the name of each target in
your grammar (.y) file.
......@@ -225,21 +225,21 @@ class BisonParser(object):
#@+node:attributes
# ---------------------------------------
# override these if you need to
# command and options for running yacc/bison, except for filename arg
bisonCmd = ["bison", "-d", "-v", '-t']
bisonFile = "tmp.y"
bisonCFile = "tmp.tab.c"
bisonHFile = "tmp.tab.h" # name of header file generated by bison cmd
bisonCFile1 = "tmp.bison.c" # c output file from bison gets renamed to this
bisonHFile1 = "tokens.h" # bison-generated header file gets renamed to this
flexCmd = ["flex", ] # command and options for running [f]lex, except for filename arg
flexFile = "tmp.l"
flexCFile = "lex.yy.c"
flexCFile1 = "tmp.lex.c" # c output file from lex gets renamed to this
cflags_pre = ['-fPIC'] # = CFLAGS added before all arguments.
......@@ -247,27 +247,27 @@ class BisonParser(object):
buildDirectory = './' # Directory used to store the generated / compiled files.
debugSymbols = 1 # Add debugging symbols to the binary files.
verbose = 0
file = None # default to sys.stdin
last = None # last parsed target, top of parse tree
lasterror = None # gets set if there was an error
keepfiles = 0 # set to 1 to keep temporary engine build files
bisonEngineLibName = None # defaults to 'modulename-engine'
defaultNodeClass = BisonNode # class to use by default for creating new parse nodes
#@-node:attributes
#@+node:__init__
def __init__(self, **kw):
"""
Abstract representation of parser
Keyword arguments:
- read - a callable accepting an int arg (nbytes) and returning a string,
default is this class' read() method
......@@ -285,7 +285,7 @@ class BisonParser(object):
read = kw.get('read', None)
if read:
self.read = read
fileobj = kw.get('file', None)
if fileobj:
if type(fileobj) == type(""):
......@@ -296,34 +296,31 @@ class BisonParser(object):
self.file = fileobj
else:
self.file = sys.stdin
nodeClass = kw.get('defaultNodeClass', None)
if nodeClass:
self.defaultNodeClass = nodeClass
self.verbose = kw.get('verbose', 0)
if kw.has_key('keepfiles'):
self.keepfiles = kw['keepfiles']
# if engine lib name not declared, invent ont
if not self.bisonEngineLibName:
self.bisonEngineLibName = self.__class__.__module__ + "-parser"
# get an engine
self.engine = ParserEngine(self)
#@-node:__init__
#@+node:__getattr__
def __getitem__(self, idx):
return self.last[idx]
#@-node:__getattr__
#@+node:_handle
def _handle(self, targetname, option, names, values):
"""
Callback which receives a target from parser, as a targetname
and list of term names and values.
Tries to dispatch to on_TargetName() methods if they exist,
otherwise wraps the target in a BisonNode object
"""
......@@ -347,20 +344,17 @@ class BisonParser(object):
if self.verbose:
print "no handler for %s, using default" % targetname
self.last = BisonNode(targetname, option=option, names=names, values=values)
# reset any resulting errors (assume they've been handled)
#self.lasterror = None
# assumedly the last thing parsed is at the top of the tree
return self.last
#@-node:_handle
#@+node:run
def run(self, **kw):
"""
Runs the parser, and returns the top-most parse target.
Keywords:
- file - either a string, comprising a file to open and read input from, or
a Python file object
......@@ -368,7 +362,7 @@ class BisonParser(object):
"""
if self.verbose:
print "Parser.run: calling engine"
# grab keywords
fileobj = kw.get('file', self.file)
if type(fileobj) == type(""):
......@@ -380,47 +374,44 @@ class BisonParser(object):
else:
filename = None
fileobj = None
read = kw.get('read', self.read)
debug = kw.get('debug', 0)
# back up existing attribs
oldfile = self.file
oldread = self.read
# plug in new ones, if given
if fileobj:
self.file = fileobj
if read:
self.read = read
# do the parsing job, spew if error
self.lasterror = None
self.engine.runEngine(debug)
if self.lasterror:
#print "Got error: %s" % repr(self.error)
if filename != None:
raise ParserSyntaxError("%s:%d: '%s' near '%s'" % ((filename,) + self.lasterror))
else:
raise ParserSyntaxError("Line %d: '%s' near '%s'" % self.lasterror)
# restore old values
self.file = oldfile
self.read = oldread
if self.verbose:
print "Parser.run: back from engine"
return self.last
#@-node:run
#@+node:read
def read(self, nbytes):
"""
Override this in your subclass, if you desire.
Arguments:
- nbytes - the maximum length of the string which you may return.
DO NOT return a string longer than this, or else Bad Things will
......@@ -433,56 +424,48 @@ class BisonParser(object):
if self.verbose:
print "Parser.read: got %s bytes" % len(bytes)
return bytes
#@-node:read
#@+node:_error
def _error(self, linenum, msg, tok):
print "Parser: line %s: syntax error '%s' before '%s'" % (linenum, msg, tok)
#@-node:_error
#@+node:error
def error(self, value):
"""
Return the result of this method from a handler to notify a syntax error
"""
self.lasterror = value
return BisonError(value)
#@-node:error
#@+node:toxml
def toxml(self):
"""
Serialises the parse tree and returns it as a raw xml string
"""
return self.last.toxml()
#@-node:toxml
#@+node:toxmldoc
def toxmldoc(self):
"""
Returns an xml.dom.minidom.Document object containing the parse tree
"""
return self.last.toxmldoc()
#@-node:toxmldoc
#@+node:toprettyxml
def toprettyxml(self):
"""
Returns a human-readable xml representation of the parse tree
"""
return self.last.toprettyxml()
#@-node:toprettyxml
#@+node:loadxml
def loadxml(self, raw, namespace=None):
"""
Loads a parse tree from raw xml text
Stores it in the '.last' attribute, which is where the root node
of parsed text gets stored
Arguments:
- raw - string containing the raw xml
- namespace - a dict or module object, where the node classes required for
reconstituting the parse tree, can be found
Returns:
- root node object of reconstituted parse tree
"""
......@@ -490,26 +473,24 @@ class BisonParser(object):
tree = self.loadxmldoc(doc, namespace)
self.last = tree
return tree
#@-node:loadxml
#@+node:loadxmldoc
def loadxmldoc(self, xmldoc, namespace=None):
"""
Returns a reconstituted parse tree, loaded from an
xml.dom.minidom.Document instance
Arguments:
- xmldoc - an xml.dom.minidom.Document instance
- namespace - a dict from which to find the classes needed
to translate the document into a tree of parse nodes
"""
return self.loadxmlobj(xmldoc.childNodes[0], namespace)
#@-node:loadxmldoc
#@+node:loadxmlobj
def loadxmlobj(self, xmlobj, namespace=None):
"""
Returns a node object, being a parse tree, reconstituted from an
xml.dom.minidom.Element object
Arguments:
- xmlobj - an xml.dom.minidom.Element instance
- namespace - a namespace from which the node classes
......@@ -520,34 +501,34 @@ class BisonParser(object):
namespace = namespace.__dict__
elif namespace == None:
namespace = globals()
objname = xmlobj.tagName
classname = objname + "_Node"
classobj = namespace.get(classname, None)
namespacekeys = namespace.keys()
# barf if node is not a known parse node or token
if (not classobj) and objname not in self.tokens:
raise Exception("Cannot reconstitute %s: can't find required node class or token %s" % (
objname, classname))
if classobj:
nodeobj = classobj()
# add the attribs
for k,v in xmlobj.attributes.items():
setattr(nodeobj, k, v)
else:
nodeobj = None
#print "----------------"
#print "objname=%s" % repr(objname)
#print "classname=%s" % repr(classname)
#print "classobj=%s" % repr(classobj)
#print "nodeobj=%s" % repr(nodeobj)
# now add the children
for child in xmlobj.childNodes:
#print "%s attributes=%s" % (child, child.attributes.items())
......@@ -560,22 +541,15 @@ class BisonParser(object):
# it's a token
childobj = child.childNodes[0].nodeValue
#print "got token %s=%s" % (childname, childobj)
nodeobj.names.append(childname)
nodeobj.values.append(childobj)
# done
return nodeobj
#@-node:loadxmlobj
#@+node:_globals
def _globals(self):
return globals().keys()
#@-node:_globals
#@-others
#@-node:class BisonParser
#@+node:bisonToPython
def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
"""
Rips the rules, tokens and precedences from a bison file, and the
......@@ -610,7 +584,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
except:
raise Exception("Cannot open lex file %s" % lexfileName)
# break up into the three '%%'-separated sections
# break up into the three '%%'-separated sections
try:
prologue, rulesRaw, epilogue = rawBison.split("\n%%\n")
except:
......@@ -618,13 +592,13 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
"File %s is not a properly formatted bison file"
" (needs 3 sections separated by %%%%" % (bisonfileName)
)
# --------------------------------------
# process prologue
prologue = prologue.split("%}")[-1].strip() # ditch the C code
prologue = re.sub("\\n([\t ]+)", " ", prologue) # join broken lines
#prologueLines = [line.strip() for line in prologue.split("\n")]
lines = prologue.split("\n")
tmp = []
......@@ -685,7 +659,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
terms = tmp
rules.append((tgt, terms))
# now we have our rulebase, we can churn out our skeleton Python file
pyfile.write("\n".join([
'#!/usr/bin/env python',
......@@ -737,7 +711,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
'# ------------------------------------------------------',
'\n',
]))
# now spit out class decs for every parse target
for target, options in rules:
tmp = []
......@@ -763,7 +737,7 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
'\n',
]))
# start churning out the class dec
pyfile.write("\n".join([
'class Parser(BisonParser):',
......@@ -942,8 +916,3 @@ def bisonToPython(bisonfileName, lexfileName, pyfileName, generateClasses=0):
'',
'',
]))
#@-node:bisonToPython
#@-others
#@-node:@file src/python/bison.py
#@-leo
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment