Created Importer object that handles file imports in begin section of cached text files.

importer.py

+import web
+import re
+import os.path
+
+from cache import Cache, to_dir, assert_file_exists
+
+
+ALIAS_PREFIX = '@'
+
+
+def separate_imports(content):
+    """
+    Separate the import section from the rest of the content of a file. Returns
+    a tuple with a list of imports, and the remaning part of the string.
+
+    Imports are located in the beginning of a file, and can take the following
+    forms:
+    # No imports
+    content
+
+    # Single file import
+    import foo
+
+    content
+
+    # Multiple files import
+    import foo, bar
+
+    content
+
+    # Multiline import
+    import foo, bar, \
+           baz
+
+    content
+    """
+    content = content.lstrip()
+    imports = []
+
+    comma = re.compile(' *, *(?:\\\\\r?\n *)?')
+    file_list = '\w+(?:%s\w+?)*' % comma.pattern
+    m = re.match('^import (%s)(?:(?:\r?\n)+(.*))?$' % file_list, content, re.M)
+
+    if m:
+        imports = comma.split(m.group(1))
+        content = m.group(2)
+
+        if not content:
+            content = ''
+
+    return imports, content
+
+
+class Importer(Cache):
+    def __init__(self, root, extension=None):
+        Cache.__init__(self)
+        self.root = to_dir(root + '/')
+        self.aliases = {}
+
+        if extension:
+            self.extension = '.' + extension
+        else:
+            self.extension = ''
+
+    def set_alias(self, name, path):
+        """
+        Add an alias for the path to a file, relative to the importer's root
+        directory. Aliases are used to shorten the names of imports. E.g. using
+        "@jquery" in an import could point to "static/js/jquery-min-1.7.1.js".
+        """
+        path = self.root + path + self.extension
+        assert_file_exists(path)
+        self.aliases[name] = path
+
+    def create_full_paths(self, files, relative_file=None):
+        """
+        Create full paths out of a file list:
+        1. Replace any aliases.
+        2. Look for direct siblings of the loaded file, if any (relative_file).
+        3. Look in the root folder of the Import object.
+        """
+        replaced = []
+        alias_len = len(ALIAS_PREFIX)
+
+        if relative_file:
+            relative_dir = to_dir(relative_file)
+
+        for i, path in enumerate(files):
+            if path[:alias_len] == ALIAS_PREFIX:
+                # Alias syntax is used, assert that the alias exists
+                alias = path[alias_len:]
+
+                if alias not in self.aliases:
+                    raise ValueError('Alias "%s" has not been set.' % alias)
+
+                # Alias exists, translate is to a full path
+                replaced.append(self.aliases[alias])
+            else:
+                if relative_file:
+                    relative_path = relative_dir + path + self.extension
+
+                    if os.path.exists(relative_path):
+                        # Relative import
+                        replaced.append(relative_path)
+                        continue
+
+                # Import from root
+                path = self.root + path + self.extension
+                assert_file_exists(path)
+                replaced.append(path)
+
+        return replaced
+
+    def concatenate(self, files):
+        files = self.create_full_paths(files)
+        map(assert_file_exists, files)
+        self.loaded = []
+        self.import_map = {}
+        concat = ''
+
+        for path in files:
+            f = open(path, 'r')
+            raw = f.read()
+            f.close()
+
+            # Parse imports from file content
+            imports, content = separate_imports(raw)
+            self.import_map.update({path: imports})
+
+            # Only add imported files that have not been loaded yet
+            new_imports = filter(lambda i: i not in self.loaded, imports)
+
+            # Prepend imports that have not been loaded yet before the file's
+            # content
+            if new_imports:
+                new_imports = self.create_full_paths(new_imports, path)
+
+                # Check if there is a recursive import
+                for f in new_imports:
+                    if f == path:
+                        raise ImportError('Recursive import in file "%s".' % f)
+
+                    if f in self.import_map and path in self.import_map[f]:
+                        raise ImportError('Recursive import in of "%s" in ' \
+                                          + 'file "%s".' % (f, path))
+
+                concat += self.concatenate(new_imports)
+
+            concat += content
+            self.loaded += path
+
+        return concat
+
+    def content(self):
+        return self.concatenate(self.files)

tests/test_importer.py

+import unittest
+import os
+import shutil
+
+from importer import separate_imports, Importer
+
+
+d = os.path.realpath('') + '/'
+
+
+class TestImporter(unittest.TestCase):
+    def setUp(self):
+        self.none = 'no imports'
+        self.single = 'import bar'
+        self.multiple = 'import bar, baz\nfoobar'
+        self.multiline = 'import bar, \\\n       baz\nfoobarbaz'
+
+        if os.path.exists('foo'):
+            shutil.rmtree('foo')
+
+        os.mkdir('foo')
+
+        for name in ('none', 'single', 'multiple', 'multiline'):
+            f = open('foo/' + name, 'w')
+            f.write(self.__getattribute__(name))
+            f.close()
+
+        self.importer = Importer('foo')
+
+    def tearDown(self):
+        shutil.rmtree('foo')
+
+    def test___init__(self):
+        self.assertEqual(self.importer.root, d + 'foo/')
+        self.assertEqual(self.importer.extension, '')
+
+    def test___init___extension(self):
+        self.assertEqual(Importer('foo', extension='txt').extension, '.txt')
+
+    def test_separate_imports(self):
+        self.assertEqual(separate_imports(self.none), ([], self.none))
+        self.assertEqual(separate_imports(self.single), (['bar'], ''))
+        self.assertEqual(separate_imports(self.multiple),
+                         (['bar', 'baz'], 'foobar'))
+        self.assertEqual(separate_imports(self.multiline),
+                         (['bar', 'baz'], 'foobarbaz'))
+
+    def test_create_full_paths_simple(self):
+        self.assertEqual(self.importer.create_full_paths([]), [])
+        self.assertEqual(self.importer.create_full_paths(['none']),
+                         [d + 'foo/none'])
+
+    def test_create_full_paths_alias(self):
+        self.importer.set_alias('bar', 'none')
+        self.assertEqual(self.importer.create_full_paths(['@bar']),
+                         [d + 'foo/none'])
+        self.assertRaises(ValueError, self.importer.create_full_paths,
+                          ['@foo'])
+
+    def test_create_full_paths_relative_file(self):
+        importer = Importer('.')
+        self.assertEqual(importer.create_full_paths(['none'], d + 'foo/single'),
+                         [d + 'foo/none'])