[feature] Backward conversion #70

- Parsers
- command line utility
- updated docs
- some long test file have been split
- openwrt converters have been split

Closes #70

Author: nemesifier

bin/netjsonconfig

@@ -34,9 +34,9 @@ parser = argparse.ArgumentParser(description=description,
 config = parser.add_argument_group('input')
 
 config.add_argument('--config', '-c',
-                    required=True,
                     action='store',
                     type=str,
+                    default=None,
                     help='config file or string, must be valid NetJSON DeviceConfiguration')
 
 config.add_argument('--templates', '-t',
@@ -46,25 +46,32 @@ config.add_argument('--templates', '-t',
                     default=[],
                     help='list of template config files or strings separated by space')
 
+config.add_argument('--native', '-n',
+                    action='store',
+                    type=str,
+                    default=None,
+                    help='path to native configuration file or archive')
+
 output = parser.add_argument_group('output')
 
 output.add_argument('--backend', '-b',
                     required=True,
-                    choices=['openwrt', 'openwisp'],
+                    choices=['openwrt', 'openwisp', 'openvpn'],
                     action='store',
                     type=str,
-                    help='Configuration backend: openwrt or openwisp')
+                    help='Configuration backend')
 
 output.add_argument('--method', '-m',
                     required=True,
-                    choices=['render', 'generate', 'write', 'validate'],
+                    choices=['render', 'generate', 'write', 'validate', 'json'],
                     action='store',
                     help='Backend method to use. '
                          '"render" returns the configuration in text format; '
                          '"generate" returns a tar.gz archive as output; '
                          '"write" is like generate but writes to disk; '
                          '"validate" validates the combination of config '
-                         'and templates passed in input; ')
+                         'and templates passed in input; '
+                         '"json" returns NetJSON output; ')
 
 output.add_argument('--args', '-a',
                     nargs='*',  # zero or more
@@ -85,14 +92,15 @@ debug.add_argument('--version', '-v',
                    version=netjsonconfig.get_version())
 
 
-def _load(config):
+def _load(config, read=True):
     """
     if config argument does not look like a JSON string
     try to read the contents of a file
     """
     if not config.strip().startswith('{'):
         try:
-            return open(config, 'r').read()
+            f = open(config, 'r')
+            return f.read() if read else f
         except IOError:
             print('netjsonconfig: cannot open "{0}": '
                   'file not found'.format(config))
@@ -146,20 +154,32 @@ def print_output(output):
 
 
 args = parser.parse_args()
-config = _load(args.config)
+if args.config:
+    config = _load(args.config)
+elif args.native:
+    native = _load(args.native, read=False)
+else:
+    print('Expected one of the following parameters: "config" or "native"; none found')
+    sys.exit(1)
 templates = [_load(template) for template in args.templates]
 context = dict(os.environ)
 method = args.method
 method_arguments = parse_method_arguments(args.args)
 
 backends = {
     'openwrt': netjsonconfig.OpenWrt,
-    'openwisp': netjsonconfig.OpenWisp
+    'openwisp': netjsonconfig.OpenWisp,
+    'openvpn': netjsonconfig.OpenVpn
 }
 
 backend_class = backends[args.backend]
 try:
-    instance = backend_class(config, templates=templates, context=context)
+    options = dict(templates=templates, context=context)
+    if args.config:
+        options['config'] = config
+    else:
+        options['native'] = native
+    instance = backend_class(**options)
 except TypeError as e:
     print('netjsonconfig: invalid JSON passed in config or templates')
     sys.exit(2)

docs/source/backends/openwrt.rst

@@ -18,7 +18,15 @@ Initialization
 
 .. automethod:: netjsonconfig.OpenWrt.__init__
 
-Initialization example:
+If you are unsure about the meaning of the initalization parameters,
+read about the following basic concepts:
+
+    * :ref:`configuration_dictionary`
+    * :ref:`backend`
+    * :ref:`template`
+    * :ref:`context`
+
+Initialization example (forward conversion):
 
 .. code-block:: python
 
@@ -30,12 +38,13 @@ Initialization example:
         }
     })
 
-If you are unsure about the meaning of the initalization parameters,
-read about the following basic concepts:
+Initialization example (backward conversion):
 
-    * :ref:`configuration_dictionary`
-    * :ref:`template`
-    * :ref:`context`
+.. code-block:: python
+
+    from netjsonconfig import OpenWrt
+
+    router = OpenWrt(native=open('./openwrt-config.tar.gz'))
 
 Render method
 -------------
@@ -176,6 +185,24 @@ Example:
 
 Will write the configuration archive in ``/tmp/dhcp-router.tar.gz``.
 
+Parse method
+------------
+
+.. automethod:: netjsonconfig.OpenWrt.parse
+
+This method is automatically called when initializing the backend
+with the ``native`` argument:
+
+.. code-block:: python
+
+    from netjsonconfig import OpenWrt
+
+    router = OpenWrt(native=open('./openwrt-config.tar.gz'))
+
+The argument passed to ``native`` can be a string containing a dump obtained via
+``uci export``, or a file object (real file or ``BytesIO`` instance) representing
+a configuration archive in tar.gz format typically used in OpenWRT/LEDE.
+
 JSON method
 -----------
 

docs/source/general/basics.rst

@@ -4,12 +4,11 @@ Basic concepts
 
 .. include:: ../_github.rst
 
-
 Before starting, let's quickly introduce the main concepts used in netjsonconfig:
 
 * :ref:`configuration_dictionary`: python dictionary representing the configuration of a router
-* :ref:`backend`: python class used to process the configuration and generate the final
-  router configuration
+* :ref:`backend`: python class used to convert the *configuration dictionary* to the format used
+  natively by a router firmware and vice versa
 * :ref:`schema`: each backend has a `JSON-Schema <http://json-schema.org>`_ which
   defines the useful configuration options that the backend is able to process
 * :ref:`validation`: the configuration is validated against its JSON-Schema before
@@ -22,16 +21,16 @@ Before starting, let's quickly introduce the main concepts used in netjsonconfig
 
 .. _configuration_dictionary:
 
-Configuration dictionary
-------------------------
+NetJSON configuration dictionary
+--------------------------------
 
 *netjsonconfig* is an implementation of the `NetJSON <http://netjson.org>`_ format,
 more specifically the ``DeviceConfiguration`` object, therefore to understand the
 configuration format that the library uses to generate the final router configurations
 it is essential to read at least the relevant `DeviceConfiguration section in the
 NetJSON RFC <http://netjson.org/rfc.html#rfc.section.5>`_.
 
-Here it is a simple NetJSON DeviceConfiguration object:
+Here it is a simple *NetJSON DeviceConfiguration* object represented with a python dictionary:
 
 .. code-block:: python
 
@@ -94,14 +93,16 @@ From now on we will use the term *configuration dictionary* to refer to
 Backend
 -------
 
-A backend is a python class used to process the *configuration dictionary* and
-generate the final router configuration, each supported firmware or opearting system
-will have its own backend and third parties can write their own custom backends.
+A backend is a python class used to convert the *configuration dictionary* to the format used
+natively by the router (forward conversion, from NetJSON to native) and vice versa (backward conversion,
+from native to NetJSON), each supported firmware or opearting system will have its own backend
+and third parties can write their own custom backends.
 
 The current implemented backends are:
 
  * :doc:`OpenWrt </backends/openwrt>`
  * :doc:`OpenWisp </backends/openwisp>` (based on the ``OpenWrt`` backend)
+ * :doc:`OpenVpn </backends/openvpn>` (custom backend implementing only OpenVPN configuration)
 
 Example initialization of ``OpenWrt`` backend:
 
@@ -126,6 +127,21 @@ Example initialization of ``OpenWrt`` backend:
         ]
     })
 
+Each backend will implement **parsers**, **renderers** and **converters** to accomplish its
+configuration generation or parsing goals.
+
+The process is best explained with the following diagram:
+
+.. image:: ../images/netjsonconfig-backward-conversion.svg
+
+**Converters** take care of converting between *NetJSON* and the intermediate data structure
+(and vice versa).
+
+**Renderers** take care of rendering the intermediate data structure to the native format.
+
+**Parsers** perform the opposite operation of ``Renderers``: they take care of parsing native format and
+build the intermediate data structure.
+
 .. _schema:
 
 Schema

docs/source/general/commandline_utility.rst

@@ -11,10 +11,11 @@ languages.
 Check out the available options yourself with::
 
     $ netjsonconfig --help
-    usage: netjsonconfig [-h] --config CONFIG
-                         [--templates [TEMPLATES [TEMPLATES ...]]] --backend
-                         {openwrt,openwisp} --method {render,generate,write,validate}
-                         [--args [ARGS [ARGS ...]]] [--verbose] [--version]
+    usage: netjsonconfig [-h] [--config CONFIG]
+                     [--templates [TEMPLATES [TEMPLATES ...]]]
+                     [--native NATIVE] --backend {openwrt,openwisp,openvpn}
+                     --method {render,generate,write,validate,json}
+                     [--args [ARGS [ARGS ...]]] [--verbose] [--version]
 
     Converts a NetJSON DeviceConfiguration object to native router configurations.
     Exhaustive documentation is available at: http://netjsonconfig.openwisp.org/
@@ -29,30 +30,34 @@ Check out the available options yourself with::
       --templates [TEMPLATES [TEMPLATES ...]], -t [TEMPLATES [TEMPLATES ...]]
                             list of template config files or strings separated by
                             space
+      --native NATIVE, -n NATIVE
+                            path to native configuration file or archive
 
     output:
-      --backend {openwrt,openwisp}, -b {openwrt,openwisp}
-                            Configuration backend: openwrt or openwisp
-      --method {render,generate,write}, -m {render,generate,write, validate}
+      --backend {openwrt,openwisp,openvpn}, -b {openwrt,openwisp,openvpn}
+                            Configuration backend
+      --method {render,generate,write,validate,json}, -m {render,generate,write,validate,json}
                             Backend method to use. "render" returns the
                             configuration in text format; "generate" returns a
                             tar.gz archive as output; "write" is like generate but
                             writes to disk; "validate" validates the combination
                             of config and templates passed in input;
-
+                            "json" returns NetJSON output:
       --args [ARGS [ARGS ...]], -a [ARGS [ARGS ...]]
                             Optional arguments that can be passed to methods
 
     debug:
       --verbose             verbose output
       --version, -v         show program's version number and exit
 
-
 Here's the common use cases explained::
 
    # generate tar.gz from a NetJSON DeviceConfiguration object and save its output to a file
    netjsonconfig --config config.json --backend openwrt --method generate > config.tar.gz
 
+   # convert an OpenWRT tar.gz to NetJSON and print to standard output (with 4 space indentation)
+   netjsonconfig --native config.tar.gz --backend openwrt --method json -a indent="    "
+
    # use write configuration archive to disk in /tmp/routerA.tar.gz
    netjsonconfig --config config.json --backend openwrt --method write --args name=routerA path=/tmp/