Skip to content

T
trs
Commit aa09d9d7 authored by Sander Mathijs van Veen's avatar Sander Mathijs van Veen
Browse files
Options

Implemented CONCAT grammar, code cleanup and fixed unit tests.

parent e1cd6997
Hide whitespace changes
Inline Side-by-side
Showing with 173 additions and 110 deletions
src/parser.py
View file @ aa09d9d7
......@@ -18,6 +18,11 @@ sys.path.insert(1, PYBISON_PYREX)
from bison import BisonParser, ParserSyntaxError
# Check for n-ary operator in child nodes
def combine(op, n):
return n.nodes if n.title() == op else [n]
class Parser(BisonParser):
"""
Implements the calculator parser. Grammar rules are defined in the method
......@@ -34,7 +39,7 @@ class Parser(BisonParser):
# of tokens of the lex script.
tokens = ['NUMBER', 'IDENTIFIER',
'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'POW',
'LPAREN', 'RPAREN', 'COMMA',
'LPAREN', 'RPAREN', 'COMMA', 'CONCAT_POW',
'NEWLINE', 'QUIT', 'RAISE']
# ------------------------------
......@@ -87,7 +92,7 @@ class Parser(BisonParser):
# as a shell. In that case, it is useful that the shell prints the
# output of the evaluation.
if self.interactive and values[1]:
print 'result:', values[1]
print values[1]
return values[1]
......@@ -107,85 +112,101 @@ class Parser(BisonParser):
"""
exp : NUMBER
| IDENTIFIER
| exp PLUS exp
| exp MINUS exp
| exp TIMES exp
| exp DIVIDE exp
| MINUS exp %prec NEG
| exp POW exp
| LPAREN exp RPAREN
| symbolic
| unary
| binary
| concat
"""
# rule: NUMBER
if option == 0:
if option == 0: # rule: NUMBER
# TODO: A bit hacky, this achieves long integers and floats.
value = float(values[0]) if '.' in values[0] else int(values[0])
return Leaf(value)
# rule: IDENTIFIER
if option == 1:
if option == 1: # rule: IDENTIFIER
return Leaf(values[0])
# rule: LPAREN exp RPAREN
if option == 8:
if option == 2: # rule: LPAREN exp RPAREN
return values[1]
# rule: symbolic
if option == 9:
if option in [3, 4, 5]: # rule: unary | binary | concat
return values[0]
# Check for n-ary operator in child nodes
combine = lambda op, n: n.nodes if n.title() == op else [n]
raise ParserSyntaxError('Unsupported option %d in target "%s".'
% (option, target))
# rule: exp PLUS exp
if option == 2:
return Node('+', *(combine('+', values[0]) + combine('+', values[2])))
def on_unary(self, target, option, names, values):
"""
unary : MINUS exp %prec NEG
"""
# rule: exp MINUS expo
if option == 3:
return Node('-', *(combine('-', values[0]) + combine('-', values[2])))
if option == 0: # rule: NEG exp
return Node('-', values[1])
# rule: exp TIMES expo
if option == 4:
return Node('*', *(combine('*', values[0]) + combine('*', values[2])))
raise ParserSyntaxError('Unsupported option %d in target "%s".'
% (option, target))
# rule: exp DIVIDE expo
if option == 5:
return Node('/', values[0], values[2])
def on_binary(self, target, option, names, values):
"""
binary : exp PLUS exp
| exp MINUS exp
| exp TIMES exp
| exp DIVIDE exp
| exp POW exp
"""
# rule: NEG expo
if option == 6:
return Node('-', values[1])
if option == 0: # rule: exp PLUS exp
return Node('+', *(combine('+', values[0])
+ combine('+', values[2])))
if option == 1: # rule: exp MINUS exp
return Node('-', *(combine('-', values[0])
+ combine('-', values[2])))
if option == 2: # rule: exp TIMES exp
return Node('*', *(combine('*', values[0])
+ combine('*', values[2])))
if option == 3: # rule: exp DIVIDE exp
return Node('/', values[0], values[2])
# rule: exp POW expo
if option == 7:
if option == 4: # rule: exp POW exp
return Node('^', values[0], values[2])
raise ParserSyntaxError('Unsupported option %d in target "%s".'
% (option, target))
def on_symbolic(self, target, option, names, values):
def on_concat(self, option, target, names, values):
"""
symbolic : NUMBER IDENTIFIER
| IDENTIFIER IDENTIFIER
| symbolic IDENTIFIER
| IDENTIFIER NUMBER
concat : exp IDENTIFIER
| exp NUMBER
| exp LPAREN exp RPAREN
| exp CONCAT_POW
| CONCAT_POW
"""
# rule: NUMBER IDENTIFIER
# rule: IDENTIFIER IDENTIFIER
# rule: symbolic IDENTIFIER
if option in [0, 1, 2]:
# 4x -> 4*x
# a b -> a * b
# a b c -> (a * b) * c
node = Node('*', Leaf(values[0]), Leaf(values[1]))
return node
# rule: IDENTIFIER NUMBER
if option in [0, 1]: # rule: exp IDENTIFIER | exp NUMBER
# NOTE: xy -> x*y
# NOTE: (x)4 -> x*4
val = int(values[1]) if option == 1 else values[1]
return Node('*', *(combine('*', values[0]) + [Leaf(val)]))
if option == 2: # rule: exp LPAREN exp RPAREN
# NOTE: x(y) -> x*(y)
return Node('*', *(combine('*', values[0])
+ combine('*', values[2])))
if option == 3:
# x4 -> x^4
return Node('^', Leaf(values[0]), Leaf(values[1]))
# NOTE: x4 -> x^4
identifier, exponent = list(values[1])
node = Node('^', Leaf(identifier), Leaf(int(exponent)))
return Node('*', values[0], node)
if option == 4:
# NOTE: x4 -> x^4
identifier, exponent = list(values[0])
return Node('^', Leaf(identifier), Leaf(int(exponent)))
raise ParserSyntaxError('Unsupported option %d in target "%s".'
% (option, target))
......@@ -196,19 +217,22 @@ class Parser(BisonParser):
lexscript = r"""
%{
//int yylineno = 0;
#include <stdio.h>
#include <string.h>
#include "Python.h"
#define YYSTYPE void *
#include "tokens.h"
extern void *py_parser;
extern void (*py_input)(PyObject *parser, char *buf, int *result, int max_size);
#define returntoken(tok) yylval = PyString_FromString(strdup(yytext)); return (tok);
#define YY_INPUT(buf,result,max_size) { (*py_input)(py_parser, buf, &result, max_size); }
extern void (*py_input)(PyObject *parser, char *buf, int *result,
int max_size);
#define returntoken(tok) \
yylval = PyString_FromString(strdup(yytext)); return (tok);
#define YY_INPUT(buf,result,max_size) { \
(*py_input)(py_parser, buf, &result, max_size); \
}
%}
%%
[a-zA-Z][0-9]+ { returntoken(CONCAT_POW); }
[0-9]+ { returntoken(NUMBER); }
[a-zA-Z] { returntoken(IDENTIFIER); }
"(" { returntoken(LPAREN); }
......@@ -224,7 +248,8 @@ class Parser(BisonParser):
[ \t\v\f] {}
[\n] {yylineno++; returntoken(NEWLINE); }
. { printf("unknown char %c ignored, yytext=0x%lx\n", yytext[0], yytext); /* ignore bad chars */}
. { printf("unknown char %c ignored, yytext=%p\n",
yytext[0], yytext); /* ignore bad chars */}
%%
......
tests/parser.py
View file @ aa09d9d7
import sys
from external.graph_drawing.graph import generate_graph
from external.graph_drawing.line import generate_line
class ParserWrapper(object):
def __init__(self, base_class, **kwargs):
self.input_buffer = []
self.last_buffer = ''
self.input_position = 0
self.closed = False
self.verbose = kwargs.get('verbose', False)
# Overwrite parser read() method
def read(nbytes):
buf = ''
try:
buf = self.input_buffer[self.input_position]
self.parser = base_class(file=self, read=self.read, **kwargs)
if self.verbose:
print 'read:', buf
except IndexError:
return ''
def readline(self, nbytes=False):
return self.read(nbytes)
self.input_position += 1
def read(self, nbytes=False):
if len(self.last_buffer) >= nbytes:
buf = self.last_buffer[:nbytes]
self.last_buffer = self.last_buffer[nbytes:]
return buf
self.parser = base_class(**kwargs)
self.parser.read = read
buf = self.last_buffer
try:
buf += self.input_buffer[self.input_position]
if self.verbose:
print 'read:', buf
self.input_position += 1
except IndexError:
self.closed = True
return ''
self.last_buffer = buf[nbytes:]
return buf
def close(self):
self.closed = True
self.input_position = len(self.input_buffer)
def run(self, input_buffer, *args, **kwargs):
map(self.append, input_buffer)
return self.parser.run(*args, **kwargs)
def append(self, input):
self.closed = False
self.input_buffer.append(input + '\n')
......@@ -70,5 +88,19 @@ def run_expressions(base_class, expressions, keepfiles=1, fail=True,
print >>sys.stderr, 'error: %s = %s, but expected: %s' \
% (exp, str(res), str(out))
if not silent and hasattr(res, 'nodes'):
print >>sys.stderr, 'result graph:'
print >>sys.stderr, generate_graph(res)
print >>sys.stderr, 'expected graph:'
print >>sys.stderr, generate_graph(out)
if fail:
raise
def graph(parser, *exp, **kwargs):
return generate_graph(ParserWrapper(parser, **kwargs).run(exp))
def line(parser, *exp, **kwargs):
return generate_line(ParserWrapper(parser, **kwargs).run(exp))
tests/test_calc.py
View file @ aa09d9d7
import unittest
from src.calc import Parser
from src.parser import Parser
from src.node import ExpressionNode as N, ExpressionLeaf as L
from tests.parser import ParserWrapper, run_expressions
class TestCalc(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def test_constructor(self):
assert ParserWrapper(Parser, keepfiles=1).run(['1+4']) == 5.0
assert ParserWrapper(Parser).run(['1+4']) \
== N('+', L(1), L(4))
def test_basic_on_exp(self):
expressions = [('4', 4.0),
('3+4', 7.0),
('3-4', -1.0),
('3/4', .75),
('-4', -4.0),
('3^4', 81.0),
('(4)', 4.0)]
expressions = [('4', L(4)),
('3+4', N('+', L(3), L(4))),
('3-4', N('-', L(3), L(4))),
('3/4', N('/', L(3), L(4))),
('-4', N('-', L(4))),
('3^4', N('^', L(3), L(4))),
('(2)', L(2))]
run_expressions(Parser, expressions)
def test_infinity(self):
expressions = [('2^3000', 2**3000),
('2^-3000', 0.0)]
expressions = [('2^3000', N('^', L(2), L(3000))),
('2^-3000', N('^', L(2), N('-', L(3000))))]
# ('2^99999999999', None),
# ('2^-99999999999', 0.0)]
run_expressions(Parser, expressions)
def test_concat_easy(self):
expressions = [
('xy', N('*', L('x'), L('y'))),
('2x', N('*', L(2), L('x'))),
('x4', N('^', L('x'), L(4))),
('xy4', N('*', L('x'), N('^', L('y'), L(4)))),
('(x)4', N('*', L('x'), L(4))),
('(3+4)2', N('*', N('+', L(3), L(4)), L(2))),
]
run_expressions(Parser, expressions)
def test_concat_intermediate(self):
expressions = [
('(3+4)(5+7)', N('*', N('+', L(3), L(4)),
N('+', L(5), L(7)))),
('(a+b)(c+d)', N('*', N('+', L('a'), L('b')),
N('+', L('c'), L('d')))),
('a+b(c+d)', N('+', L('a'), N('*', L('b'),
N('+', L('c'), L('d'))))),
('ab(c)d', N('*', L('a'), L('b'), L('c'), L('d'))),
#('ab(c)d', N('*', L('a'), N('*', L('b'),
# N('*', L('c'), L('d'))))),
]
run_expressions(Parser, expressions)
tests/test_parser.py
View file @ aa09d9d7
# vim: set fileencoding=utf-8 :
import unittest
from external.graph_drawing.graph import generate_graph
from external.graph_drawing.line import generate_line
from src.parser import Parser
from src.node import ExpressionNode as Node, ExpressionLeaf as Leaf
from tests.parser import ParserWrapper, run_expressions
def graph(*exp, **kwargs):
return generate_graph(ParserWrapper(Parser, **kwargs).run(exp))
def line(*exp, **kwargs):
return generate_line(ParserWrapper(Parser, **kwargs).run(exp))
from tests.parser import ParserWrapper, run_expressions, line, graph
class TestParser(unittest.TestCase):
......@@ -26,11 +15,11 @@ class TestParser(unittest.TestCase):
run_expressions(Parser, [('a', Leaf('a'))])
def test_graph(self):
assert graph('4a') == ("""
assert graph(Parser, '4a') == ("""
*
╭┴╮
4 a
""").replace('\n ', '\n')[1:-1]
def test_line(self):
self.assertEqual(line('4a'), '4 * a')
self.assertEqual(line(Parser, '4a'), '4 * a')
tests/test_variables.py
View file @ aa09d9d7
......@@ -7,12 +7,6 @@ from sympy import Symbol, symbols
class TestVariables(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def test_addition(self):
expressions = [('5 + 5', 5 + 5)]
run_expressions(Parser, expressions)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment