diff --git a/layers/admin.py b/layers/admin.py index 63612b6..81d85f2 100644 --- a/layers/admin.py +++ b/layers/admin.py @@ -210,11 +210,38 @@ def get_formset(self, request, obj, **kwargs): pass return formset +@admin.action(description="Export Theme(s) for migration") +def export_theme_details(self, request, queryset): + from django.http import HttpResponse + + if not queryset.exists(): + self.message_user(request, "Please select at least one theme to export.") + return + + theme_id_list = [] + fixture_rows = [] + seen_rows = set() + for theme in queryset: + theme_id_list.append(theme.id) + for row in theme.to_export_dict(): + row_key = (row['model'], row['source_pk']) + if row_key not in seen_rows: + seen_rows.add(row_key) + fixture_rows.append(row) + + theme_ids = "_".join(str(id) for id in theme_id_list) + response = HttpResponse(json.dumps(fixture_rows), content_type='application/geo+json') + filename = f"themes_{theme_ids}.json" + response['Content-Disposition'] = f'attachment; filename="{filename}"' + return response + + class ThemeAdmin(ImportExportMixin,admin.ModelAdmin): list_display = ('display_name', 'name', 'order', 'date_modified', 'is_top_theme', 'primary_site', 'preview_site') search_fields = ['display_name', 'name',] form = ThemeForm inlines = [ThemeParentInline, ExistingChildInline, ChildInline] + actions = [export_theme_details] fieldsets = ( ('BASIC INFO', { diff --git a/layers/models.py b/layers/models.py index 0ab71b3..eeac361 100644 --- a/layers/models.py +++ b/layers/models.py @@ -552,6 +552,12 @@ def shortDict(self, site=None, order=None): cache.set(cache_label, layers_dict, 60*60*24*7) return layers_dict + def to_export_dict(self): + from layers.serializers import ThemeExportFixtureSerializer + + serializer = ThemeExportFixtureSerializer(self) + return serializer.to_representation(self) + def __str__(self): return "{} [T-{}]".format(self.name, self.pk) diff --git a/layers/serializers.py b/layers/serializers.py index 58499f0..701e9d0 100644 --- a/layers/serializers.py +++ b/layers/serializers.py @@ -6,6 +6,7 @@ from layers.fixture_contract import build_node, build_ref from layers.models import Theme, Layer, ChildOrder, Companionship, LayerWMS, LayerArcREST, LayerArcFeatureService, LayerVector, LayerXYZ, AttributeInfo, LookupInfo, MultilayerDimension, MultilayerDimensionValue, MultilayerAssociation from rest_framework import serializers +from rest_framework.utils.serializer_helpers import ReturnList #need to add catalog html to shared_layer_fields after adding it to subtheme serializer and to layer model shared_layer_fields = ["id", "name", "uuid", "type", "url", "proxy_url", "is_disabled", "disabled_message", "opacity", "show_legend", "legend", "legend_title", "legend_subtitle", "description", "overview", "data_url", @@ -427,6 +428,92 @@ def to_representation(self, instance): return fixture_rows +class ThemeExportFixtureSerializer(serializers.Serializer): + @property + def data(self): + if not hasattr(self, '_data'): + self._data = ReturnList( + self.to_representation(self.instance), + serializer=self, + ) + return self._data + + def _to_ref(self, instance): + return build_ref(instance=instance) + + def _serialize_value(self, value): + if value is None or isinstance(value, (str, int, float, bool)): + return value + return str(value) + + def _model_fields(self, instance, excluded_fields): + return { + field.name: self._serialize_value(field.value_from_object(instance)) + for field in instance._meta.concrete_fields + if field.name not in excluded_fields + } + + def _to_row(self, instance, fields, relations=None): + return build_node( + model=instance._meta.label_lower, + source_pk=instance.pk, + uuid_value=getattr(instance, 'uuid', None), + fields=fields, + relations=relations, + ) + + def to_representation(self, instance): + fixture_rows = [] + seen_theme_pks = set() + seen_child_order_pks = set() + seen_row_keys = set() + theme_queue = [instance] + + def append_row(row): + row_key = (row['model'], row['source_pk']) + if row_key not in seen_row_keys: + seen_row_keys.add(row_key) + fixture_rows.append(row) + + while theme_queue: + theme = theme_queue.pop(0) + if theme.pk in seen_theme_pks: + continue + seen_theme_pks.add(theme.pk) + + append_row(self._to_row( + theme, + self._model_fields(theme, {'id', 'site'}), + )) + + child_orders = ChildOrder.objects.filter(parent_theme=theme).order_by('order', 'pk') + for child_order in child_orders: + content_object = child_order.content_object + if content_object is None or child_order.pk in seen_child_order_pks: + continue + seen_child_order_pks.add(child_order.pk) + + append_row(self._to_row( + child_order, + self._model_fields(child_order, { + 'id', 'parent_theme', 'content_type', 'object_id', + }), + { + 'parent_theme': self._to_ref(theme), + 'content_object': self._to_ref(content_object), + }, + )) + + if isinstance(content_object, Theme): + if content_object.pk not in seen_theme_pks: + theme_queue.append(content_object) + elif isinstance(content_object, Layer): + for row in LayerExportFixtureSerializer().to_representation(content_object): + append_row(row) + + return fixture_rows + + def get_companion_layers(obj): if hasattr(obj, 'layer'): layer_instance = obj.layer diff --git a/layers/tests/test_models.py b/layers/tests/test_models.py index b58fa09..ffc3e61 100644 --- a/layers/tests/test_models.py +++ b/layers/tests/test_models.py @@ -2,14 +2,14 @@ from django.utils import timezone from datetime import date from layers.models import Theme, Layer, MultilayerAssociation, MultilayerDimension, MultilayerDimensionValue, Companionship, LayerWMS, LayerArcREST, LayerArcFeatureService, LayerVector, LayerXYZ, ChildOrder, AttributeInfo, LookupInfo -from layers.serializers import ThemeSerializer, LayerWMSSerializer, CompanionLayerSerializer, LayerArcRESTSerializer, LayerArcFeatureServiceSerializer, LayerXYZSerializer, LayerVectorSerializer, SubThemeSerializer, ChildOrderSerializer, LayerExportSerializer, AttributeInfoExportSerializer, LookupInfoExportSerializer, LayerWMSExportSerializer, LayerArcRESTExportSerializer, LayerArcFeatureServiceExportSerializer, LayerVectorExportSerializer, LayerXYZExportSerializer +from layers.serializers import ThemeSerializer, ThemeExportFixtureSerializer, LayerWMSSerializer, CompanionLayerSerializer, LayerArcRESTSerializer, LayerArcFeatureServiceSerializer, LayerXYZSerializer, LayerVectorSerializer, SubThemeSerializer, ChildOrderSerializer, LayerExportSerializer, AttributeInfoExportSerializer, LookupInfoExportSerializer, LayerWMSExportSerializer, LayerArcRESTExportSerializer, LayerArcFeatureServiceExportSerializer, LayerVectorExportSerializer, LayerXYZExportSerializer from layers.views import get_portal_catalog_map from collections.abc import Collection import json from django.contrib.sites.models import Site from django.contrib.contenttypes.models import ContentType from layers.fixture_contract import NODE_FIELDS_KEY, NODE_MODEL_KEY, NODE_RELATIONS_KEY, NODE_SOURCE_PK_KEY, NODE_UUID_KEY -from layers.admin import export_layer_details +from layers.admin import export_layer_details, export_theme_details from rest_framework import serializers from unittest.mock import Mock # request to get data from live site, mung it and make it into v2 @@ -365,7 +365,7 @@ def test_layer_export_concurrent_layers(self): self.assertEqual(len(shared_companion_rows), 1) def test_layer_export_fixture_contains_attribute_infos_followed_by_layer(self): - + create_data = { 'name': 'Export Fixture Layer', 'layer_type': 'WMS', @@ -533,6 +533,186 @@ def test_layer_export_fixture_includes_lookup_and_specific_instance_rows_for_vec ], ) + +class ThemeExportFixtureSerializerTest(TestCase): + def _rows_for_model(self, fixture_data, model_label): + return [row for row in fixture_data if row[NODE_MODEL_KEY] == model_label] + + def _assert_refers_to(self, relation, instance): + self.assertEqual( + relation, + { + NODE_MODEL_KEY: instance._meta.label_lower, + NODE_SOURCE_PK_KEY: instance.pk, + NODE_UUID_KEY: str(instance.uuid), + }, + ) + + def test_theme_export_single_theme(self): + parent_theme = Theme.objects.create(name='Parent Theme', display_name='Parent Theme') + child_theme = Theme.objects.create(name='Child Theme', display_name='Child Theme') + layer_a = Layer.objects.create(name='Layer A', layer_type='WMS') + layer_b = Layer.objects.create(name='Layer B', layer_type='WMS') + + ChildOrder.objects.create(parent_theme=parent_theme, content_object=layer_a, order=1) + ChildOrder.objects.create(parent_theme=parent_theme, content_object=child_theme, order=2) + ChildOrder.objects.create(parent_theme=child_theme, content_object=layer_a, order=1) + ChildOrder.objects.create(parent_theme=child_theme, content_object=layer_b, order=2) + + response = export_theme_details(Mock(), Mock(), Theme.all_objects.filter(pk=parent_theme.pk)) + fixture_data = json.loads(response.content) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.theme')), 2) + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.childorder')), 4) + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.layer')), 2) + + def test_theme_export_concurrent_themes(self): + parent_theme = Theme.objects.create(name='Parent Theme', display_name='Parent Theme') + child_theme = Theme.objects.create(name='Child Theme', display_name='Child Theme') + third_theme = Theme.objects.create(name='Third Theme', display_name='Third Theme') + layer_a = Layer.objects.create(name='Layer A', layer_type='WMS') + layer_b = Layer.objects.create(name='Layer B', layer_type='WMS') + layer_c = Layer.objects.create(name='Layer C', layer_type='WMS') + + ChildOrder.objects.create(parent_theme=parent_theme, content_object=layer_a, order=1) + ChildOrder.objects.create(parent_theme=parent_theme, content_object=child_theme, order=2) + ChildOrder.objects.create(parent_theme=child_theme, content_object=layer_a, order=1) + ChildOrder.objects.create(parent_theme=child_theme, content_object=layer_b, order=2) + ChildOrder.objects.create(parent_theme=third_theme, content_object=layer_a, order=1) + ChildOrder.objects.create(parent_theme=third_theme, content_object=layer_c, order=2) + + selected_themes = Theme.all_objects.filter( + pk__in=[parent_theme.pk, child_theme.pk, third_theme.pk], + ).order_by('pk') + response = export_theme_details(Mock(), Mock(), selected_themes) + fixture_data = json.loads(response.content) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.theme')), 3) + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.childorder')), 6) + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.layer')), 3) + + def test_theme_export_fixture_serializes_recursive_children_and_deduplicates(self): + root_theme = Theme.objects.create(name='Root Theme', display_name='Root Theme') + child_theme = Theme.objects.create(name='Child Theme', display_name='Child Theme') + shared_layer = Layer.objects.create(name='Shared Layer', layer_type='WMS') + + root_child_theme_order = ChildOrder.objects.create( + parent_theme=root_theme, + content_object=child_theme, + order=1, + ) + root_layer_order = ChildOrder.objects.create( + parent_theme=root_theme, + content_object=shared_layer, + order=2, + ) + child_layer_order = ChildOrder.objects.create( + parent_theme=child_theme, + content_object=shared_layer, + order=1, + ) + + fixture_data = ThemeExportFixtureSerializer(root_theme).data + theme_rows = self._rows_for_model(fixture_data, 'layers.theme') + child_order_rows = self._rows_for_model(fixture_data, 'layers.childorder') + layer_rows = self._rows_for_model(fixture_data, 'layers.layer') + + self.assertEqual(len(theme_rows), 2) + self.assertEqual(len(child_order_rows), 3) + self.assertEqual(len(layer_rows), 1) + self.assertEqual( + set(theme_rows[0][NODE_FIELDS_KEY]), + { + field.name + for field in Theme._meta.concrete_fields + if field.name not in {'id', 'site'} + }, + ) + self.assertNotIn('site', theme_rows[0][NODE_FIELDS_KEY]) + + self._assert_refers_to( + child_order_rows[0][NODE_RELATIONS_KEY]['parent_theme'], + root_theme, + ) + self._assert_refers_to( + child_order_rows[0][NODE_RELATIONS_KEY]['content_object'], + child_theme, + ) + self._assert_refers_to( + child_order_rows[1][NODE_RELATIONS_KEY]['content_object'], + shared_layer, + ) + self._assert_refers_to( + child_order_rows[2][NODE_RELATIONS_KEY]['content_object'], + shared_layer, + ) + self.assertEqual( + child_order_rows[0][NODE_FIELDS_KEY]['order'], + root_child_theme_order.order, + ) + self.assertEqual( + child_order_rows[1][NODE_FIELDS_KEY]['order'], + root_layer_order.order, + ) + self.assertEqual( + child_order_rows[2][NODE_FIELDS_KEY]['order'], + child_layer_order.order, + ) + + def test_theme_export_fixture_stops_at_self_referential_child_order(self): + theme = Theme.objects.create(name='Self Referencing Theme', display_name='Self Referencing Theme') + child_order = ChildOrder.objects.create( + parent_theme=theme, + content_object=theme, + order=1, + ) + + fixture_data = ThemeExportFixtureSerializer(theme).data + + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.theme')), 1) + child_order_rows = self._rows_for_model(fixture_data, 'layers.childorder') + self.assertEqual(len(child_order_rows), 1) + self._assert_refers_to( + child_order_rows[0][NODE_RELATIONS_KEY]['parent_theme'], + theme, + ) + self._assert_refers_to( + child_order_rows[0][NODE_RELATIONS_KEY]['content_object'], + theme, + ) + self.assertEqual(child_order_rows[0][NODE_FIELDS_KEY]['order'], child_order.order) + + def test_theme_export_fixture_deduplicates_layer_with_multiple_child_orders(self): + theme = Theme.objects.create(name='Repeated Layer Theme', display_name='Repeated Layer Theme') + layer = Layer.objects.create(name='Repeated Layer', layer_type='WMS') + first_child_order = ChildOrder.objects.create( + parent_theme=theme, + content_object=layer, + order=1, + ) + second_child_order = ChildOrder.objects.create( + parent_theme=theme, + content_object=layer, + order=2, + ) + + fixture_data = ThemeExportFixtureSerializer(theme).data + + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.theme')), 1) + self.assertEqual(len(self._rows_for_model(fixture_data, 'layers.layer')), 1) + child_order_rows = self._rows_for_model(fixture_data, 'layers.childorder') + self.assertEqual(len(child_order_rows), 2) + self.assertEqual( + [row[NODE_FIELDS_KEY]['order'] for row in child_order_rows], + [first_child_order.order, second_child_order.order], + ) + for row in child_order_rows: + self._assert_refers_to(row[NODE_RELATIONS_KEY]['content_object'], layer) + + + class AttributeInfoExportSerializerTest(TestCase): def test_attribute_info_export_contains_expected_fields_with_appropriate_types(self):