Implemented CONCAT grammar, code cleanup and fixed unit tests.

src/parser.py

@@ -18,6 +18,11 @@ sys.path.insert(1, PYBISON_PYREX)
 
 from bison import BisonParser, ParserSyntaxError
 
+
+# Check for n-ary operator in child nodes
+def combine(op, n):
+    return n.nodes if n.title() == op else [n]
+
 class Parser(BisonParser):
     """
     Implements the calculator parser. Grammar rules are defined in the method
@@ -34,7 +39,7 @@ class Parser(BisonParser):
     # of tokens of the lex script.
     tokens = ['NUMBER', 'IDENTIFIER',
               'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'POW',
-              'LPAREN', 'RPAREN', 'COMMA',
+              'LPAREN', 'RPAREN', 'COMMA', 'CONCAT_POW',
               'NEWLINE', 'QUIT', 'RAISE']
 
     # ------------------------------
@@ -87,7 +92,7 @@ class Parser(BisonParser):
             # as a shell. In that case, it is useful that the shell prints the
             # output of the evaluation.
             if self.interactive and values[1]:
-                print 'result:', values[1]
+                print values[1]
 
             return values[1]
 
@@ -107,85 +112,101 @@ class Parser(BisonParser):
         """
         exp : NUMBER
             | IDENTIFIER
-            | exp PLUS exp
-            | exp MINUS exp
-            | exp TIMES exp
-            | exp DIVIDE exp
-            | MINUS exp %prec NEG
-            | exp POW exp
             | LPAREN exp RPAREN
-            | symbolic
+            | unary
+            | binary
+            | concat
         """
 
-        # rule: NUMBER
-        if option == 0:
+        if option == 0:  # rule: NUMBER
             # TODO: A bit hacky, this achieves long integers and floats.
             value = float(values[0]) if '.' in values[0] else int(values[0])
             return Leaf(value)
 
-        # rule: IDENTIFIER
-        if option == 1:
+        if option == 1:  # rule: IDENTIFIER
             return Leaf(values[0])
 
-        # rule: LPAREN exp RPAREN
-        if option == 8:
+        if option == 2:  # rule: LPAREN exp RPAREN
             return values[1]
 
-        # rule: symbolic
-        if option == 9:
+        if option in [3, 4, 5]:  # rule: unary | binary | concat
             return values[0]
 
-        # Check for n-ary operator in child nodes
-        combine = lambda op, n: n.nodes if n.title() == op else [n]
+        raise ParserSyntaxError('Unsupported option %d in target "%s".'
+                                % (option, target))
 
-        # rule: exp PLUS exp
-        if option == 2:
-            return Node('+', *(combine('+', values[0]) + combine('+', values[2])))
+    def on_unary(self, target, option, names, values):
+        """
+        unary : MINUS exp %prec NEG
+        """
 
-        # rule: exp MINUS expo
-        if option == 3:
-            return Node('-', *(combine('-', values[0]) + combine('-', values[2])))
+        if option == 0:  # rule: NEG exp
+            return Node('-', values[1])
 
-        # rule: exp TIMES expo
-        if option == 4:
-            return Node('*', *(combine('*', values[0]) + combine('*', values[2])))
+        raise ParserSyntaxError('Unsupported option %d in target "%s".'
+                                % (option, target))
 
-        # rule: exp DIVIDE expo
-        if option == 5:
-            return Node('/', values[0], values[2])
+    def on_binary(self, target, option, names, values):
+        """
+        binary : exp PLUS exp
+               | exp MINUS exp
+               | exp TIMES exp
+               | exp DIVIDE exp
+               | exp POW exp
+        """
 
-        # rule: NEG expo
-        if option == 6:
-            return Node('-', values[1])
+        if option == 0:  # rule: exp PLUS exp
+            return Node('+', *(combine('+', values[0])
+                               + combine('+', values[2])))
+
+        if option == 1:  # rule: exp MINUS exp
+            return Node('-', *(combine('-', values[0])
+                               + combine('-', values[2])))
+
+        if option == 2:  # rule: exp TIMES exp
+            return Node('*', *(combine('*', values[0])
+                               + combine('*', values[2])))
+
+        if option == 3:  # rule: exp DIVIDE exp
+            return Node('/', values[0], values[2])
 
-        # rule: exp POW expo
-        if option == 7:
+        if option == 4:  # rule: exp POW exp
             return Node('^', values[0], values[2])
 
         raise ParserSyntaxError('Unsupported option %d in target "%s".'
                                 % (option, target))
 
-    def on_symbolic(self, target, option, names, values):
+
+    def on_concat(self, option, target, names, values):
         """
-        symbolic : NUMBER IDENTIFIER
-                 | IDENTIFIER IDENTIFIER
-                 | symbolic IDENTIFIER
-                 | IDENTIFIER NUMBER
+        concat : exp IDENTIFIER
+               | exp NUMBER
+               | exp LPAREN exp RPAREN
+               | exp CONCAT_POW
+               | CONCAT_POW
         """
-        # rule: NUMBER IDENTIFIER
-        # rule: IDENTIFIER IDENTIFIER
-        # rule: symbolic IDENTIFIER
-        if option in [0, 1, 2]:
-            # 4x -> 4*x
-            # a b -> a * b
-            # a b c -> (a * b) * c
-            node = Node('*', Leaf(values[0]), Leaf(values[1]))
-            return node
-
-        # rule: IDENTIFIER NUMBER
+
+        if option in [0, 1]:  # rule: exp IDENTIFIER | exp NUMBER
+            # NOTE: xy -> x*y
+            # NOTE: (x)4 -> x*4
+            val = int(values[1]) if option == 1 else values[1]
+            return Node('*', *(combine('*', values[0]) + [Leaf(val)]))
+
+        if option == 2:  # rule: exp LPAREN exp RPAREN
+            # NOTE: x(y) -> x*(y)
+            return Node('*', *(combine('*', values[0])
+                              + combine('*', values[2])))
+
         if option == 3:
-            # x4 -> x^4
-            return Node('^', Leaf(values[0]), Leaf(values[1]))
+            # NOTE: x4 -> x^4
+            identifier, exponent = list(values[1])
+            node = Node('^', Leaf(identifier), Leaf(int(exponent)))
+            return Node('*', values[0], node)
+
+        if option == 4:
+            # NOTE: x4 -> x^4
+            identifier, exponent = list(values[0])
+            return Node('^', Leaf(identifier), Leaf(int(exponent)))
 
         raise ParserSyntaxError('Unsupported option %d in target "%s".'
                                 % (option, target))
@@ -196,19 +217,22 @@ class Parser(BisonParser):
     lexscript = r"""
     %{
     //int yylineno = 0;
-    #include <stdio.h>
-    #include <string.h>
     #include "Python.h"
     #define YYSTYPE void *
     #include "tokens.h"
     extern void *py_parser;
-    extern void (*py_input)(PyObject *parser, char *buf, int *result, int max_size);
-    #define returntoken(tok) yylval = PyString_FromString(strdup(yytext)); return (tok);
-    #define YY_INPUT(buf,result,max_size) { (*py_input)(py_parser, buf, &result, max_size); }
+    extern void (*py_input)(PyObject *parser, char *buf, int *result,
+                            int max_size);
+    #define returntoken(tok) \
+        yylval = PyString_FromString(strdup(yytext)); return (tok);
+    #define YY_INPUT(buf,result,max_size) { \
+        (*py_input)(py_parser, buf, &result, max_size); \
+    }
     %}
 
     %%
 
+    [a-zA-Z][0-9]+ { returntoken(CONCAT_POW); }
     [0-9]+    { returntoken(NUMBER); }
     [a-zA-Z]  { returntoken(IDENTIFIER); }
     "("       { returntoken(LPAREN); }
@@ -224,7 +248,8 @@ class Parser(BisonParser):
 
     [ \t\v\f] {}
     [\n]      {yylineno++; returntoken(NEWLINE); }
-    .         { printf("unknown char %c ignored, yytext=0x%lx\n", yytext[0], yytext); /* ignore bad chars */}
+    .         { printf("unknown char %c ignored, yytext=%p\n",
+                yytext[0], yytext); /* ignore bad chars */}
 
     %%
 

tests/parser.py

 import sys
 
+from external.graph_drawing.graph import generate_graph
+from external.graph_drawing.line import generate_line
+
 
 class ParserWrapper(object):
 
     def __init__(self, base_class, **kwargs):
         self.input_buffer = []
+        self.last_buffer = ''
         self.input_position = 0
+        self.closed = False
 
         self.verbose = kwargs.get('verbose', False)
 
-        # Overwrite parser read() method
-        def read(nbytes):
-            buf = ''
-
-            try:
-                buf = self.input_buffer[self.input_position]
+        self.parser = base_class(file=self, read=self.read, **kwargs)
 
-                if self.verbose:
-                    print 'read:', buf
-            except IndexError:
-                return ''
+    def readline(self, nbytes=False):
+        return self.read(nbytes)
 
-            self.input_position += 1
+    def read(self, nbytes=False):
 
+        if len(self.last_buffer) >= nbytes:
+            buf = self.last_buffer[:nbytes]
+            self.last_buffer = self.last_buffer[nbytes:]
             return buf
 
-        self.parser = base_class(**kwargs)
-        self.parser.read = read
+        buf = self.last_buffer
+
+        try:
+            buf += self.input_buffer[self.input_position]
+
+            if self.verbose:
+                print 'read:', buf
+
+            self.input_position += 1
+        except IndexError:
+            self.closed = True
+            return ''
+
+        self.last_buffer = buf[nbytes:]
+        return buf
 
+    def close(self):
+        self.closed = True
+        self.input_position = len(self.input_buffer)
 
     def run(self, input_buffer, *args, **kwargs):
         map(self.append, input_buffer)
         return self.parser.run(*args, **kwargs)
 
     def append(self, input):
+        self.closed = False
         self.input_buffer.append(input + '\n')
 
 
@@ -70,5 +88,19 @@ def run_expressions(base_class, expressions, keepfiles=1, fail=True,
                 print >>sys.stderr, 'error: %s = %s, but expected: %s' \
                                     % (exp, str(res), str(out))
 
+            if not silent and hasattr(res, 'nodes'):
+                print >>sys.stderr, 'result graph:'
+                print >>sys.stderr, generate_graph(res)
+                print >>sys.stderr, 'expected graph:'
+                print >>sys.stderr, generate_graph(out)
+
             if fail:
                 raise
+
+
+def graph(parser, *exp, **kwargs):
+    return generate_graph(ParserWrapper(parser, **kwargs).run(exp))
+
+
+def line(parser, *exp, **kwargs):
+    return generate_line(ParserWrapper(parser, **kwargs).run(exp))

tests/test_calc.py

 import unittest
 
-from src.calc import Parser
+from src.parser import Parser
+from src.node import ExpressionNode as N, ExpressionLeaf as L
 from tests.parser import ParserWrapper, run_expressions
 
 
 class TestCalc(unittest.TestCase):
 
-    def setUp(self):
-        pass
-
-    def tearDown(self):
-        pass
-
     def test_constructor(self):
-        assert ParserWrapper(Parser, keepfiles=1).run(['1+4']) == 5.0
+        assert ParserWrapper(Parser).run(['1+4']) \
+                == N('+', L(1), L(4))
 
     def test_basic_on_exp(self):
-        expressions = [('4', 4.0),
-                       ('3+4', 7.0),
-                       ('3-4', -1.0),
-                       ('3/4', .75),
-                       ('-4', -4.0),
-                       ('3^4', 81.0),
-                       ('(4)', 4.0)]
+        expressions = [('4',   L(4)),
+                       ('3+4', N('+', L(3), L(4))),
+                       ('3-4', N('-', L(3), L(4))),
+                       ('3/4', N('/', L(3), L(4))),
+                       ('-4',  N('-', L(4))),
+                       ('3^4', N('^', L(3), L(4))),
+                       ('(2)', L(2))]
 
         run_expressions(Parser, expressions)
 
     def test_infinity(self):
-        expressions = [('2^3000', 2**3000),
-                       ('2^-3000', 0.0)]
+        expressions = [('2^3000',  N('^', L(2), L(3000))),
+                       ('2^-3000', N('^', L(2), N('-', L(3000))))]
         #               ('2^99999999999', None),
         #               ('2^-99999999999', 0.0)]
 
         run_expressions(Parser, expressions)
+
+    def test_concat_easy(self):
+        expressions = [
+                       ('xy',     N('*', L('x'), L('y'))),
+                       ('2x',     N('*', L(2), L('x'))),
+                       ('x4',     N('^', L('x'), L(4))),
+                       ('xy4',    N('*', L('x'), N('^', L('y'), L(4)))),
+                       ('(x)4',   N('*', L('x'), L(4))),
+                       ('(3+4)2', N('*', N('+', L(3), L(4)), L(2))),
+                      ]
+
+        run_expressions(Parser, expressions)
+
+    def test_concat_intermediate(self):
+        expressions = [
+                       ('(3+4)(5+7)', N('*', N('+', L(3), L(4)),
+                                             N('+', L(5), L(7)))),
+                       ('(a+b)(c+d)', N('*', N('+', L('a'), L('b')),
+                                             N('+', L('c'), L('d')))),
+                       ('a+b(c+d)',   N('+', L('a'), N('*', L('b'),
+                                             N('+', L('c'), L('d'))))),
+                       ('ab(c)d',   N('*', L('a'), L('b'), L('c'), L('d'))),
+                       #('ab(c)d',   N('*', L('a'), N('*', L('b'),
+                       #                              N('*', L('c'), L('d'))))),
+                      ]
+
+        run_expressions(Parser, expressions)

tests/test_parser.py

 # vim: set fileencoding=utf-8 :
 import unittest
 
-from external.graph_drawing.graph import generate_graph
-from external.graph_drawing.line import generate_line
-
 from src.parser import Parser
 from src.node import ExpressionNode as Node, ExpressionLeaf as Leaf
-from tests.parser import ParserWrapper, run_expressions
-
-
-def graph(*exp, **kwargs):
-    return generate_graph(ParserWrapper(Parser, **kwargs).run(exp))
-
-
-def line(*exp, **kwargs):
-    return generate_line(ParserWrapper(Parser, **kwargs).run(exp))
+from tests.parser import ParserWrapper, run_expressions, line, graph
 
 
 class TestParser(unittest.TestCase):
@@ -26,11 +15,11 @@ class TestParser(unittest.TestCase):
         run_expressions(Parser, [('a', Leaf('a'))])
 
     def test_graph(self):
-        assert graph('4a') == ("""
+        assert graph(Parser, '4a') == ("""
          *
         ╭┴╮
         4 a
         """).replace('\n        ', '\n')[1:-1]
 
     def test_line(self):
-        self.assertEqual(line('4a'), '4 * a')
+        self.assertEqual(line(Parser, '4a'), '4 * a')

tests/test_variables.py

@@ -7,12 +7,6 @@ from sympy import Symbol, symbols
 
 class TestVariables(unittest.TestCase):
 
-    def setUp(self):
-        pass
-
-    def tearDown(self):
-        pass
-
     def test_addition(self):
         expressions = [('5 + 5', 5 + 5)]
         run_expressions(Parser, expressions)