[fix] Ignored duplicate list elements when merging templates #197

Detect identical elements present in both lists
to avoid adding the duplicate to the result.

This is needed because some templates may share
one or multiple common files and these do not
not have to be duplicated.

Fixes #197

Co-authored-by: Federico Capoano <f.capoano@openwisp.io>

Author: codesankalp

netjsonconfig/utils.py

@@ -55,6 +55,13 @@ def merge_list(list1, list2, identifiers=None):
         for el in list_:
             # merge by internal python id by default
             key = id(el)
+            # Detect identical elements present in both lists
+            # avoid adding the duplicate to the result.
+            # This is needed because some templates may share
+            # one or multiple common files and these do not
+            # not have to be duplicated.
+            if counter == 2 and el in dict_map['list1'].values():
+                continue
             # if el is a dict, merge by keys specified in ``identifiers``
             if isinstance(el, dict):
                 for id_key in identifiers:

tests/openvpn/test_backend.py

@@ -886,3 +886,42 @@ def test_tls_auth_key_present(self):
         )
         client = OpenVpn(client_config)
         self.assertEqual(client.render(), self._openvpn_client_tls_auth_render)
+
+    def test_ca_same_file_path_for_same_device(self):
+        conf = {
+            "ca": "/etc/x509/ca.pem",
+            "cert": "cert.pem",
+            "dev": "tap0",
+            "dev_type": "tap",
+            "dh": "dh.pem",
+            "key": "key.pem",
+            "mode": "server",
+            "name": "test-1",
+            "proto": "udp",
+            "tls_server": True,
+        }
+        ca_file = {"path": "/etc/x509/ca.pem", "mode": "0644", "contents": "testing!"}
+        template1 = {
+            "openvpn": [{**conf}],
+            "files": [ca_file],
+        }
+        template2 = {
+            "openvpn": [
+                {
+                    **conf,
+                    "name": "test-2",
+                    "cert": "cert2.pem",
+                }
+            ],
+            "files": [ca_file],
+        }
+        client = OpenVpn(
+            {
+                "openvpn": [conf],
+            },
+            templates=[template1, template2],
+        )
+        try:
+            client.render()
+        except ValidationError:
+            self.fail('ValidationError raised!')

tests/test_utils.py

@@ -138,3 +138,12 @@ def test_merge_list_config_value(self):
 
     def test_get_copy_default(self):
         self.assertEqual('test', get_copy({}, key='hello', default='test'))
+
+    def test_merge_list_skip_duplicates(self):
+        conf1 = [{"mode": "0644", "contents": "test"}]
+        conf2 = [
+            {"mode": "0644", "contents": "test"},
+            [{"mode": "0644", "contents": "test2"}],
+        ]
+        result = merge_list(conf1, conf2)
+        self.assertEqual(result, conf2)