[feature] Added organization variables #780

Closes #780

Author: pandafy

README.rst

@@ -537,8 +537,9 @@ the order (high to low) of their precedence:
 1. `User defined device variables <#user-defined-device-variables>`_
 2. `Predefined device variables <#predefined-device-variables>`_
 3. `Group variables <#group-variables>`_
-4. `Global variables <#global-variables>`_
-5. `Template default values <#template-default-values>`_
+4. `Organization variables <#organization-variables>`_
+5. `Global variables <#global-variables>`_
+6. `Template default values <#template-default-values>`_
 
 User defined device variables
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -568,6 +569,18 @@ Variables can also be defined in `Device groups <#device-groups>`__.
 Refer the `Group configuration variables <group-configuration-variables>`_
 section for detailed information.
 
+Organization variables
+~~~~~~~~~~~~~~~~~~~~~~
+
+Variables can also be defined at the organization level.
+
+You can set the *organization variables* from the organization change page
+``/admin/openwisp_users/organization/<organization-id>/change/``, under the
+**Configuration Management Settings**.
+
+.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/organization-variables.png
+  :alt: organization variables
+
 Global variables
 ~~~~~~~~~~~~~~~~
 

openwisp_controller/config/admin.py

@@ -663,6 +663,7 @@ def _get_preview_instance(self, request):
         c.device.mac_address = request.POST.get('mac_address')
         c.device.key = request.POST.get('key')
         c.device.group_id = request.POST.get('group') or None
+        c.device.organization_id = request.POST.get('organization_id') or None
         if 'hardware_id' in request.POST:
             c.device.hardware_id = request.POST.get('hardware_id')
         return c
@@ -1087,7 +1088,8 @@ def has_delete_permission(self, request, obj):
 if getattr(app_settings, 'REGISTRATION_ENABLED', True):
 
     class ConfigSettingsForm(AlwaysHasChangedMixin, forms.ModelForm):
-        pass
+        class Meta:
+            widgets = {'context': FlatJsonWidget}
 
     class ConfigSettingsInline(admin.StackedInline):
         model = OrganizationConfigSettings

openwisp_controller/config/base/config.py

@@ -150,6 +150,11 @@ def get_cached_checksum(self):
         logger.debug(f'calculating checksum for config ID {self.pk}')
         return self.checksum
 
+    @classmethod
+    def bulk_invalidate_get_cached_checksum(cls, query_params):
+        for config in cls.objects.only('id').filter(**query_params).iterator():
+            config.get_cached_checksum.invalidate(config)
+
     @classmethod
     def get_template_model(cls):
         return cls.templates.rel.model
@@ -616,6 +621,10 @@ def get_context(self, system=False):
                     ('key', self.key),
                 ]
             )
+            config_settings = self.device._get_organization__config_settings()
+            if config_settings:
+                # Add organization variables
+                context.update(config_settings.get_context())
             if self.device._get_group():
                 # Add device group variables
                 context.update(self.device._get_group().get_context())

openwisp_controller/config/base/device.py

@@ -130,6 +130,11 @@ def _has_config(self):
     def _has_group(self):
         return hasattr(self, 'group')
 
+    def _has_organization__config_settings(self):
+        return hasattr(self, 'organization') and hasattr(
+            self.organization, 'config_settings'
+        )
+
     def _get_config_attr(self, attr):
         """
         gets property or calls method of related config object
@@ -151,6 +156,13 @@ def _get_group(self):
             return self.group
         return self.get_group_model()(device=self)
 
+    def _get_organization__config_settings(self):
+        if self._has_organization__config_settings():
+            return self.organization.config_settings
+        return load_model('config', 'OrganizationConfigSettings')(
+            organization=self.organization if hasattr(self, 'organization') else None
+        )
+
     def get_context(self):
         config = self._get_config()
         return config.get_context()
@@ -357,6 +369,14 @@ def get_default_templates(self):
     def get_config_model(cls):
         return cls._meta.get_field('config').related_model
 
+    @classmethod
+    def get_group_model(cls):
+        return cls._meta.get_field('group').related_model
+
+    @classmethod
+    def get_config_settings_model(cls):
+        return
+
     def get_temp_config_instance(self, **options):
         config = self.get_config_model()(**options)
         config.device = self

openwisp_controller/config/base/multitenancy.py

@@ -1,10 +1,15 @@
+import collections
+from copy import deepcopy
+
 import swapper
 from django.db import models
 from django.utils.translation import gettext_lazy as _
+from jsonfield import JSONField
 
 from openwisp_utils.base import KeyField, UUIDModel
 
 from ..exceptions import OrganizationDeviceLimitExceeded
+from ..tasks import bulk_invalidate_config_get_cached_checksum
 
 
 class AbstractOrganizationConfigSettings(UUIDModel):
@@ -27,6 +32,16 @@ class AbstractOrganizationConfigSettings(UUIDModel):
         verbose_name=_('shared secret'),
         help_text=_('used for automatic registration of devices'),
     )
+    context = JSONField(
+        blank=True,
+        default=dict,
+        load_kwargs={'object_pairs_hook': collections.OrderedDict},
+        dump_kwargs={'indent': 4},
+        help_text=_(
+            'This field can be used to add "Configuration Variables"' ' to the devices.'
+        ),
+        verbose_name=_('Configuration Variables'),
+    )
 
     class Meta:
         verbose_name = _('Configuration management settings')
@@ -36,6 +51,22 @@ class Meta:
     def __str__(self):
         return self.organization.name
 
+    def get_context(self):
+        return deepcopy(self.context)
+
+    def save(
+        self, force_insert=False, force_update=False, using=None, update_fields=None
+    ):
+        context_changed = False
+        if not self._state.adding:
+            db_instance = self.__class__.objects.only('context').get(id=self.id)
+            context_changed = db_instance.context != self.context
+        super().save(force_insert, force_update, using, update_fields)
+        if context_changed:
+            bulk_invalidate_config_get_cached_checksum.delay(
+                {'device__organization_id': str(self.organization_id)}
+            )
+
 
 class AbstractOrganizationLimits(models.Model):
     organization = models.OneToOneField(

openwisp_controller/config/migrations/0051_organizationconfigsettings_context.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.20 on 2023-07-22 10:49
+
+import collections
+
+import jsonfield.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('config', '0050_alter_vpnclient_unique_together'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organizationconfigsettings',
+            name='context',
+            field=jsonfield.fields.JSONField(
+                blank=True,
+                default=dict,
+                dump_kwargs={'indent': 4},
+                help_text=(
+                    'This field can be used to add "Configuration Variables"'
+                    ' to the devices.'
+                ),
+                load_kwargs={'object_pairs_hook': collections.OrderedDict},
+                verbose_name='Configuration Variables',
+            ),
+        ),
+    ]

openwisp_controller/config/static/config/js/widget.js

@@ -69,12 +69,16 @@
     },
     getDefaultValues = function (isLoading=false) {
         var templatePks = $('input[name="config-0-templates"]').attr('value'),
-            groupPk = $('#id_group').val();
+            groupPk = $('#id_group').val(),
+            orgPk = $('#id_organization').val();
         if (templatePks) {
             var payload = {pks: templatePks};
             if (groupPk) {
                 payload.group = groupPk;
             }
+            if (orgPk) {
+                payload.organization = orgPk;
+            }
             $.get(defaultValuesUrl, payload)
                 .done( function (data) {
                     updateContext(isLoading, data.default_values);
@@ -454,6 +458,9 @@
         $('#id_group').on('change', function() {
             getDefaultValues();
         });
+        $('#id_organization').on('change', function() {
+            getDefaultValues();
+        });
     });
 
     // deletes maxLength on ip address schema if address contains variable

openwisp_controller/config/tasks.py

@@ -140,3 +140,9 @@ def change_devices_templates(instance_id, model_name, **kwargs):
             backend=kwargs.pop('backend'),
             **kwargs,
         )
+
+
+@shared_task(soft_time_limit=7200)
+def bulk_invalidate_config_get_cached_checksum(query_params):
+    Config = load_model('config', 'Config')
+    Config.bulk_invalidate_get_cached_checksum(query_params)

openwisp_controller/config/tests/test_admin.py

@@ -38,6 +38,7 @@
 DeviceGroup = load_model('config', 'DeviceGroup')
 Template = load_model('config', 'Template')
 Vpn = load_model('config', 'Vpn')
+OrganizationConfigSettings = load_model('config', 'OrganizationConfigSettings')
 Ca = load_model('django_x509', 'Ca')
 Cert = load_model('django_x509', 'Cert')
 User = get_user_model()
@@ -1428,9 +1429,24 @@ def test_clone_template(self):
         self.assertEqual(LogEntry.objects.all().count(), 3)
 
     def test_get_default_values(self):
-        t1 = self._create_template(name='t1', default_values={'name1': 'test1'})
+        org = self._get_org()
+        OrganizationConfigSettings.objects.create(
+            organization=org,
+            context={
+                'name1': 'organization variable',
+                'name3': 'should not appear',
+            },
+        )
+        t1 = self._create_template(
+            name='t1', default_values={'name1': 'test1'}, organization=org
+        )
         group = self._create_device_group(
-            context={'name1': 'device group', 'name2': 'should not appear'}
+            organization=org,
+            context={
+                'name1': 'device group',
+                'name2': 'should not appear',
+                'name4': 'should not appear',
+            },
         )
         path = reverse('admin:get_default_values')
 
@@ -1459,12 +1475,43 @@ def test_get_default_values(self):
                 response_data = response.json()
                 self.assertEqual(response_data, expected)
                 self.assertNotIn('name2', response_data)
+                self.assertNotIn('name4', response_data)
+
+        with self.subTest('get default values conflicting with organization'):
+            with self.assertNumQueries(4):
+                response = self.client.get(
+                    path, {'pks': f'{t1.pk}', 'organization': str(org.pk)}
+                )
+                self.assertEqual(response.status_code, 200)
+                expected = {'default_values': {'name1': 'organization variable'}}
+                response_data = response.json()
+                self.assertEqual(response_data, expected)
+                self.assertNotIn('name3', response_data)
+
+        with self.subTest('get default values conflicting with organization and group'):
+            with self.assertNumQueries(5):
+                response = self.client.get(
+                    path,
+                    {
+                        'pks': f'{t1.pk}',
+                        'group': str(group.pk),
+                        'organization': str(org.pk),
+                    },
+                )
+                self.assertEqual(response.status_code, 200)
+                expected = {'default_values': {'name1': 'device group'}}
+                response_data = response.json()
+                self.assertEqual(response_data, expected)
+                self.assertNotIn('name2', response_data)
+                self.assertNotIn('name3', response_data)
+                self.assertNotIn('name4', response_data)
 
     def test_get_default_values_invalid_pks(self):
         path = reverse('admin:get_default_values')
         expected = {
             'template': {'error': 'invalid template pks were received'},
-            'group': {'error': 'invalid group pk were received'},
+            'group': {'error': 'invalid group pk was received'},
+            'organization': {'error': 'invalid organization pk was received'},
         }
 
         with self.subTest('test with invalid template pk'):
@@ -1482,6 +1529,11 @@ def test_get_default_values_invalid_pks(self):
             self.assertEqual(r.status_code, 400)
             self.assertEqual(r.json(), expected['group'])
 
+        with self.subTest('test with invalid organization pk'):
+            r = self.client.get(path, {'organization': 'invalid', 'pks': str(uuid4())})
+            self.assertEqual(r.status_code, 400)
+            self.assertEqual(r.json(), expected['organization'])
+
     def _test_system_context_field_helper(self, path):
         r = self.client.get(path)
         self.assertEqual(r.status_code, 200)
@@ -1524,7 +1576,6 @@ def test_no_system_context(self, *args):
         self._create_template()
         path = reverse(f'admin:{self.app_label}_template_add')
         r = self.client.get(path)
-        print(app_settings.CONTEXT)
         self.assertContains(
             r, 'There are no system defined variables available right now'
         )

openwisp_controller/config/tests/test_api.py

@@ -467,7 +467,7 @@ def test_device_download_api(self):
         d1 = self._create_device()
         self._create_config(device=d1)
         path = reverse('config_api:download_device_config', args=[d1.pk])
-        with self.assertNumQueries(6):
+        with self.assertNumQueries(8):
             r = self.client.get(path)
         self.assertEqual(r.status_code, 200)