Implemented py_callback in C and error threshold

Added error threshold to avoid infinite loops in parserengine.runengine().

Cython generated crappy and unstable code for py_callback, so I implemented the
py_callback method in C. This is easier to maintain and debug. This patch fixes
The random segfault caused by Python's garbage collector.

src/c/bison_callback.c

+/*
+ * Callback functions called by bison.
+ *
+ * The original py_callback function is removed from bison_.pyx because Cython
+ * generated crappy code for that callback. Cython's generated code caused
+ * segfaults when python triggered its garbage collection. Thus, something was
+ * wrong with references. Debugging the generated code was hard and the
+ * callbacks are part of PyBison's core, so implementing the callbacks in C
+ * instead of generating them by Cython seems the right way to go.
+ *
+ * Written januari 2012 by Sander Mathijs van Veen <smvv@kompiler.org>
+ * Copyright (c) 2012 by Sander Mathijs van Veen, all rights reserved.
+ *
+ * Released under the GNU General Public License, a copy of which should appear
+ * in this distribution in the file called 'COPYING'. If this file is missing,
+ * then you can obtain a copy of the GPL license document from the GNU website
+ * at http://www.gnu.org.
+ *
+ * This software is released with no warranty whatsoever. Use it at your own
+ * risk.
+ *
+ * If you wish to use this software in a commercial application, and wish to
+ * depart from the GPL licensing requirements, please contact the author and
+ * apply for a commercial license.
+ */
+
+#include "Python.h"
+#include "stdarg.h"
+#include <stdio.h>
+
+#define likely(x)       __builtin_expect((x),1)
+#define unlikely(x)     __builtin_expect((x),0)
+
+static PyObject *py_callback_handle_name;
+static PyObject *py_callback_hook_name;
+
+
+/*
+ * Callback function which is invoked by target handlers within the C yyparse()
+ * function. This callback function will return parser._handle's python object
+ * or, on failure, NULL is returned.
+ */
+PyObject* py_callback(PyObject *parser, char *target, int option, int nargs,
+                      ...)
+{
+    va_list ap;
+    int i;
+
+    PyObject *res;
+
+    PyObject *names = PyList_New(nargs),
+        *values = PyList_New(nargs);
+
+    va_start(ap, nargs);
+
+    // Construct the names and values list from the variable argument list.
+    for(i = 0; i < nargs; i++) {
+        PyObject *name = PyString_FromString(va_arg(ap, char *));
+        Py_INCREF(name);
+        PyList_SetItem(names, i, name);
+
+        PyObject *value = va_arg(ap, PyObject *);
+        Py_INCREF(value);
+        PyList_SetItem(values, i, value);
+    }
+
+    va_end(ap);
+
+    // Construct attribute names (only the first time)
+    if (unlikely(!py_callback_handle_name)) {
+        py_callback_handle_name = PyString_FromString("_handle");
+        // TODO: where do we Py_DECREF(handle_name) ??
+    }
+
+    if (unlikely(!py_callback_hook_name)) {
+        py_callback_hook_name = PyString_FromString("hook_handler");
+        // TODO: where do we Py_DECREF(hook_name) ??
+    }
+
+    // Call the handler with the arguments
+    PyObject *handle = PyObject_GetAttr(parser, py_callback_handle_name);
+
+    if (unlikely(!handle)) return res;
+
+    PyObject *arglist = Py_BuildValue("(siOO)", target, option, names, values);
+    if (unlikely(!arglist)) { Py_DECREF(handle); return res; }
+
+    res = PyObject_CallObject(handle, arglist);
+
+    Py_DECREF(handle);
+    Py_DECREF(arglist);
+
+    if (unlikely(!res)) return res;
+
+    // Check if the "hook_handler" callback exists
+    if (unlikely(!PyObject_HasAttr(parser, py_callback_hook_name)))
+        return res;
+
+    handle = PyObject_GetAttr(parser, py_callback_hook_name);
+
+    if (unlikely(!handle)) {
+        Py_DECREF(res);
+        return NULL;
+    }
+
+    // Call the "hook_handler" callback
+    arglist = Py_BuildValue("(siOOO)", target, option, names, values, res);
+    if (unlikely(!arglist)) { Py_DECREF(handle); return res; }
+
+    res = PyObject_CallObject(handle, arglist);
+
+    Py_DECREF(handle);
+    Py_DECREF(arglist);
+
+    return res;
+}

src/c/bison_callback.h

+#include "Python.h"
+#include "stdarg.h"
+
+PyObject* py_callback(PyObject *, char *, int option, int nargs,...);

src/pyrex/bison_.pyx

@@ -44,6 +44,11 @@ cdef extern from "stdio.h":
 cdef extern from "string.h":
     void *memcpy(void *dest, void *src, long n)
 
+# Callback function which is invoked by target handlers
+# within the C yyparse() function.
+cdef extern from "../c/bison_callback.h":
+    object py_callback(object, char *, int option, int nargs,...)
+
 cdef extern from "../c/bisondynlib.h":
     void *bisondynlib_open(char *filename)
     int bisondynlib_close(void *handle)
@@ -54,71 +59,6 @@ cdef extern from "../c/bisondynlib.h":
 
     #int bisondynlib_build(char *libName, char *includedir)
 
-# Definitions for variadic functions (e.g. py_callback).
-cdef extern from "stdarg.h":
-    ctypedef struct va_list:
-        pass
-    ctypedef struct fake_type:
-        pass
-    void va_start(va_list, int arg)
-    void* va_arg(va_list, fake_type)
-    void va_end(va_list)
-    fake_type void_type "void *"
-    fake_type str_type "char *"
-
-# Callback function which is invoked by target handlers
-# within the C yyparse() function.
-
-#import signal
-
-cdef public object py_callback(object parser, char *target, int option, \
-        int nargs, ...):
-
-    cdef int i
-    cdef va_list ap
-    va_start(ap, <int>nargs)
-
-    cdef object valobj
-    cdef void *val
-    cdef char *termname
-
-    names = PyList_New(nargs)
-    values = PyList_New(nargs)
-
-    Py_INCREF(names)
-    Py_INCREF(values)
-
-    # Construct handler's names and values list.
-    for i in range(nargs):
-        termname = <char*>va_arg(ap, str_type)
-        PyList_SetItem(names, i, termname)
-        Py_INCREF(termname)
-
-        val = <void *>va_arg(ap, void_type)
-
-        if val:
-            valobj = <object>val
-        else:
-            valobj = None
-
-        PyList_SetItem(values, i, valobj)
-        Py_INCREF(valobj)
-
-    va_end(ap)
-
-    # Set the signal handler and a timeout alarm
-    #signal.signal(signal.SIGALRM, parser.handle_timeout)
-    #signal.alarm(parser.timeout)
-
-    res = parser._handle(target, option, names, values)
-
-    #signal.alarm(0)
-
-    if hasattr(parser, 'hook_handler'):
-        res = parser.hook_handler(target, option, names, values, res)
-
-    return res
-
 # callback routine for reading input
 cdef public void py_input(object parser, char *buf, int *result, int max_size):
     cdef int buflen

src/python/bison.py

@@ -105,6 +105,8 @@ class BisonParser(object):
     # BisonNode will be used.
     default_node_class = BisonNode
 
+    error_threshold = 10
+
     def __init__(self, **kw):
         """
         Abstract representation of parser
@@ -238,6 +240,8 @@ class BisonParser(object):
         if self.verbose and self.file.closed:
             print 'Parser.run(): self.file', self.file, 'is closed'
 
+        error_count = 0
+
         # TODO: add option to fail on first error.
         while not self.file.closed:
             # do the parsing job, spew if error
@@ -246,6 +250,11 @@ class BisonParser(object):
             try:
                 self.engine.runEngine(debug)
             except Exception as e:
+                error_count += 1
+
+                if error_count > self.error_threshold:
+                    raise
+
                 self.report_last_error(filename, e)
 
             if self.verbose: