[general] Refactored backends, renderers, added converters #70

This is the first step in implementing #70.

This patch does not introduce changes of behaviour in the library,
infact tests remain unaltered, except for their names (because they
were named after renderer classes which now are a much smaller
part of the library).

This patch introduces the concept of converter classes, which are
responsible for converting the NetJSON configuration dictionary
to an intermediate data structure that is then used by renderers
to be rendered.

The backward conversion will then implement the reverse process:

Parsers will be responsible of converting native configuration
to the intermediate data structure;
Converters will have a to_netjson method that will be responsible
of converting the intermediate data structure to the NetJSON
configuration dictionary.

Author: nemesifier

…g/backends/openvpn/templates/__init__.py netjsonconfig/backends/base/__init__.pynetjsonconfig/backends/openvpn/templates/__init__.py renamed to netjsonconfig/backends/base/__init__.py netjsonconfig/backends/openvpn/templates/__init__.py renamed to netjsonconfig/backends/base/__init__.py

@@ -1,26 +1,26 @@
 import gzip
 import json
 import tarfile
+from collections import OrderedDict
 from copy import deepcopy
 from io import BytesIO
 
 import six
-from jinja2 import Environment, PackageLoader
 from jsonschema import FormatChecker, validate
 from jsonschema.exceptions import ValidationError as JsonSchemaError
 
-from ..exceptions import ValidationError
-from ..schema import DEFAULT_FILE_MODE
-from ..utils import evaluate_vars, merge_config
+from ...exceptions import ValidationError
+from ...schema import DEFAULT_FILE_MODE
+from ...utils import evaluate_vars, merge_config
 
 
 class BaseBackend(object):
     """
     Base Backend class
     """
     schema = None
-    env_path = 'netjsonconfig.backends.base'
     FILE_SECTION_DELIMITER = '# ---------- files ---------- #'
+    intermediate_data = None
 
     def __init__(self, config, templates=[], context={}):
         """
@@ -35,8 +35,6 @@ def __init__(self, config, templates=[], context={}):
         config = deepcopy(self._load(config))
         self.config = self._merge_config(config, templates)
         self.config = self._evaluate_vars(self.config, context)
-        self.env = Environment(loader=PackageLoader(self.env_path, 'templates'),
-                               trim_blocks=True)
 
     def _load(self, config):
         """
@@ -113,29 +111,23 @@ def render(self, files=True):
         :returns: string with output
         """
         self.validate()
-        output = ''
-        # iterate over the available renderers
-        # to build the target configuration
-        # from the configuration dictionary
-        for renderer_class in self.renderers:
-            renderer = renderer_class(self)
-            additional_output = renderer.render()
-            # add an additional new line
-            # to separate blocks of configuration
-            # generated by different renderers
-            if output and additional_output:
-                output += '\n'
-            # concatenate the render configuration
-            output += additional_output
-
+        # convert NetJSON config to intermediate data structure
+        if self.intermediate_data is None:
+            self.to_intermediate()
+        # render intermediate data structure into native configuration
+        renderer = self.renderer(self)
+        output = renderer.render()
+        # remove reference to renderer instance (not needed anymore)
+        del renderer
         # are we required to include
         # additional files?
         if files:
             # render additional files
             files_output = self._render_files()
             if files_output:
-                output += files_output.replace('\n\n\n', '\n\n')  # max 3 \n
-        # finally return the whole configuration
+                # max 2 new lines
+                output += files_output.replace('\n\n\n', '\n\n')
+        # return the configuration
         return output
 
     def json(self, validate=True, *args, **kwargs):
@@ -235,59 +227,22 @@ def _add_file(self, tar, name, contents, mode=DEFAULT_FILE_MODE):
         info.mode = int(mode, 8)  # permissions converted to decimal notation
         tar.addfile(tarinfo=info, fileobj=byte_contents)
 
-
-class BaseRenderer(object):
-    """
-    Renderers are used to generate specific configuration blocks.
-    """
-    block_name = None
-
-    def __init__(self, backend):
-        self.config = backend.config
-        self.env = backend.env
-        self.backend = backend
-
-    @classmethod
-    def get_name(cls):
+    def to_intermediate(self):
         """
-        Get the name of the rendered without prefix
+        Converts the NetJSON configuration dictionary (self.config)
+        to the intermediate data structure (self.intermediate_data) that will
+        be then used by the renderer class to generate the router configuration
         """
-        return str(cls.__name__).replace('Renderer', '').lower()
-
-    def cleanup(self, output):
-        """
-        Performs cleanup of output (indentation, new lines)
-
-        :param output: string representation of the client configuration
-        """
-        return output
-
-    def render(self):
-        """
-        Renders config block with jinja2 templating engine
-        """
-        # get jinja2 template
-        template_name = '{0}.jinja2'.format(self.get_name())
-        template = self.env.get_template(template_name)
-        # render template and cleanup
-        context = self.get_context()
-        output = template.render(**context)
-        return self.cleanup(output)
-
-    def get_context(self):
-        """
-        Builds context dictionary to be used in jinja2 templates
-        For every method prefixed with `__get`, such as ``_get_name``, creates an entry in
-        the context dictionary whose key is ``name`` and it's value
-        it the output of ``_get_name``
-        """
-        # get list of private methods that start with "_get_"
-        methods = [method for method in dir(self) if method.startswith('_get_')]
-        context = {}
-        # build context
-        for method in methods:
-            key = method.replace('_get_', '')
-            context[key] = getattr(self, method)()
-        # determine if all context values are empty
-        context['is_empty'] = not any(context.values())
-        return context
+        self.validate()
+        data = OrderedDict()
+        for converter_class in self.converters:
+            # skip unnecessary loop cycles
+            if not converter_class.should_run(self.config):
+                continue
+            converter = converter_class(self)
+            for item in converter.to_intermediate():
+                key, value = item
+                if value:
+                    data.setdefault(key, [])
+                    data[key] += value
+        self.intermediate_data = data

netjsonconfig/backends/base.py netjsonconfig/backends/base/backend.pynetjsonconfig/backends/base.py renamed to netjsonconfig/backends/base/backend.py

@@ -0,0 +1,27 @@
+class BaseConverter(object):
+    """
+    Base Converter class
+    Converters are used to convert a configuration dictionary
+    which represent a NetJSON object to a data structure that
+    can be easily rendered as the final router configuration
+    and vice versa.
+    """
+    netjson_key = None
+
+    def __init__(self, backend):
+        self.backend = backend
+        self.netjson = backend.config
+        self.intermediate_data = backend.intermediate_data
+
+    @classmethod
+    def should_run(cls, config):
+        """
+        Returns True if Converter should be instantiated and run
+        Used to skip processing if the configuration part related to
+        the converter is not present in the configuration dictionary.
+        """
+        netjson_key = cls.netjson_key or cls.__name__.lower()
+        return netjson_key in config
+
+    def to_intermediate(self):  # pragma: no cover
+        raise NotImplementedError()

netjsonconfig/backends/base/converter.py

@@ -0,0 +1,47 @@
+from jinja2 import Environment, PackageLoader
+
+
+class BaseRenderer(object):
+    """
+    Base Renderer class
+    Renderers are used to generate a string
+    which represents the router configuration
+    """
+    def __init__(self, backend):
+        self.config = backend.config
+        self.backend = backend
+
+    @property
+    def env_path(self):
+        return self.__module__
+
+    @property
+    def template_env(self):
+        return Environment(loader=PackageLoader(self.env_path, 'templates'), trim_blocks=True)
+
+    @classmethod
+    def get_name(cls):
+        """
+        Returns the name of the render class without its prefix
+        """
+        return str(cls.__name__).replace('Renderer', '').lower()
+
+    def cleanup(self, output):
+        """
+        Performs cleanup of output (indentation, new lines)
+
+        :param output: string representation of the client configuration
+        """
+        return output
+
+    def render(self):
+        """
+        Renders configuration by using the jinja2 templating engine
+        """
+        # get jinja2 template
+        template_name = '{0}.jinja2'.format(self.get_name())
+        template = self.template_env.get_template(template_name)
+        # render template and cleanup
+        context = getattr(self.backend, 'intermediate_data', {})
+        output = template.render(data=context)
+        return self.cleanup(output)

netjsonconfig/backends/base/renderer.py

@@ -1,23 +1,15 @@
-from copy import deepcopy
+from ...utils import get_copy, sorted_dict
+from ..base.converter import BaseConverter
 
-from ...utils import sorted_dict
-from ..base import BaseRenderer
 
+class OpenVpn(BaseConverter):
+    def to_intermediate(self):
+        result = []
+        for vpn in get_copy(self.netjson, 'openvpn'):
+            result.append(sorted_dict(self.__get_vpn(vpn)))
+        return (('openvpn', result),)
 
-class OpenVpnRenderer(BaseRenderer):
-    """
-    Produces an OpenVPN configuration string
-    """
-    def cleanup(self, output):
-        # remove indentations
-        output = output.replace('    ', '')
-        # remove last newline
-        if output.endswith('\n\n'):
-            output = output[0:-1]
-        return output
-
-    def _transform_vpn(self, vpn):
-        config = deepcopy(vpn)
+    def __get_vpn(self, config):
         skip_keys = ['script_security', 'remote']
         delete_keys = []
         # allow server_bridge to be empty and still rendered
@@ -41,10 +33,3 @@ def _transform_vpn(self, vpn):
         if 'status' not in config and 'status_version' in config:
             del config['status_version']
         return config
-
-    def _get_openvpn(self):
-        openvpn = []
-        for vpn in self.config.get('openvpn', []):
-            config = self._transform_vpn(vpn)
-            openvpn.append(sorted_dict(config))
-        return openvpn

…jsonconfig/backends/openvpn/renderers.py …sonconfig/backends/openvpn/converters.pynetjsonconfig/backends/openvpn/renderers.py renamed to netjsonconfig/backends/openvpn/converters.py netjsonconfig/backends/openvpn/renderers.py renamed to netjsonconfig/backends/openvpn/converters.py

@@ -1,17 +1,18 @@
 import re
 
-from ..base import BaseBackend
-from .renderers import OpenVpnRenderer
+from . import converters
+from ..base.backend import BaseBackend
+from .renderer import OpenVpnRenderer
 from .schema import schema
 
 
 class OpenVpn(BaseBackend):
     """
-    OpenVPN 2.3 backend
+    OpenVPN 2.x Configuration Backend
     """
     schema = schema
-    env_path = 'netjsonconfig.backends.openvpn'
-    renderers = [OpenVpnRenderer]
+    converters = [converters.OpenVpn]
+    renderer = OpenVpnRenderer
     VPN_REGEXP = re.compile('# openvpn config: ')
 
     def _generate_contents(self, tar):
@@ -26,7 +27,7 @@ def _generate_contents(self, tar):
         vpn_instances = self.VPN_REGEXP.split(text)
         if '' in vpn_instances:
             vpn_instances.remove('')
-        # for each package create a file with its contents in /etc/config
+        # create a file for each VPN
         for vpn in vpn_instances:
             lines = vpn.split('\n')
             vpn_name = lines[0]

netjsonconfig/backends/openvpn/openvpn.py

@@ -0,0 +1,14 @@
+from ..base.renderer import BaseRenderer
+
+
+class OpenVpnRenderer(BaseRenderer):
+    """
+    OpenVPN Renderer
+    """
+    def cleanup(self, output):
+        # remove indentations
+        output = output.replace('    ', '')
+        # remove last newline
+        if output.endswith('\n\n'):
+            output = output[0:-1]
+        return output

netjsonconfig/backends/openvpn/renderer.py

@@ -1,4 +1,4 @@
-{% for vpn in openvpn %}
+{% for vpn in data.openvpn %}
 # openvpn config: {{ vpn.pop('name') }}
 
     {% for key, value in vpn.items() %}

netjsonconfig/backends/openvpn/templates/openvpn.jinja2

@@ -7,10 +7,10 @@
 
 
 class OpenWisp(OpenWrt):
-    """ OpenWisp 1.x Backend """
+    """
+    OpenWISP 1.x Firmware (legacy) Configuration Backend
+    """
     schema = schema
-    openwisp_env = Environment(loader=PackageLoader('netjsonconfig.backends.openwisp', 'templates'),
-                               trim_blocks=True)
 
     def validate(self):
         self._sanitize_radios()
@@ -26,7 +26,9 @@ def _sanitize_radios(self):
             radio.setdefault('disabled', False)
 
     def _render_template(self, template, context={}):
-        template = self.openwisp_env.get_template(template)
+        openwisp_env = Environment(loader=PackageLoader(self.__module__, 'templates'),
+                                   trim_blocks=True)
+        template = openwisp_env.get_template(template)
         return template.render(**context)
 
     def _add_unique_file(self, item):
@@ -140,14 +142,6 @@ def _add_tc_script(self):
             "mode": "755"
         })
 
-    def generate(self):
-        """
-        Generates an openwisp configuration archive.
-
-        :returns: in-memory tar.gz archive, instance of ``BytesIO``
-        """
-        return super(OpenWisp, self).generate()
-
     def _generate_contents(self, tar):
         """
         Adds configuration files to tarfile instance.
@@ -160,7 +154,7 @@ def _generate_contents(self, tar):
         packages = re.split('package ', uci)
         if '' in packages:
             packages.remove('')
-        # for each package create a file with its contents in /etc/config
+        # create a file for each configuration package used
         for package in packages:
             lines = package.split('\n')
             package_name = lines[0]