diff --git a/specifyweb/backend/context/tests/test_schema_localization_import.py b/specifyweb/backend/context/tests/test_schema_localization_import.py new file mode 100644 index 00000000000..222f1b4e260 --- /dev/null +++ b/specifyweb/backend/context/tests/test_schema_localization_import.py @@ -0,0 +1,175 @@ +import json + +from django.test import Client + +from specifyweb.specify import models +from specifyweb.specify.tests.test_api import ApiTests + + +class SchemaLocalizationImportTests(ApiTests): + def setUp(self): + super().setUp() + self.container = models.Splocalecontainer.objects.create( + discipline=self.discipline, name='Accession', schematype=0 + ) + self.item = models.Splocalecontaineritem.objects.create( + container=self.container, name='accessionnumber' + ) + self.client = Client() + self.client.force_login(self.specifyuser) + self.client.cookies['collection'] = str(self.collection.id) + + def test_imports_schema_values_and_skips_unknown_entries(self): + response = self.client.post( + '/context/schema_localization_import.json', + data=json.dumps({ + 'language': 'en', + 'schema': { + 'accession': { + 'format': 'Accession', + 'name': 'Imported Accession', + 'items': { + 'accessionnumber': { + 'isHidden': True, + 'name': 'Imported Number', + 'pickListName': 'Unavailable Picklist', + 'webLinkName': 'Unavailable Web Link', + }, + 'removedfield': {'isHidden': True}, + }, + }, + 'removedtable': {'isHidden': True}, + }, + }), + content_type='application/json', + ) + + self.assertEqual(response.status_code, 200) + self.container.refresh_from_db() + self.item.refresh_from_db() + self.assertEqual(self.container.format, 'Accession') + self.assertTrue(self.item.ishidden) + self.assertIsNone(self.item.picklistname) + self.assertIsNone(self.item.weblinkname) + self.assertEqual( + models.Splocaleitemstr.objects.get( + containername=self.container, language='en' + ).text, + 'Imported Accession', + ) + self.assertEqual( + models.Splocaleitemstr.objects.get( + itemname=self.item, language='en' + ).text, + 'Imported Number', + ) + + def test_import_updates_existing_countryless_string(self): + string = models.Splocaleitemstr.objects.create( + containername=self.container, + language='en', + country='', + text='Existing Accession', + ) + + response = self.client.post( + '/context/schema_localization_import.json', + data=json.dumps({ + 'language': 'en', + 'schema': {'accession': {'name': 'Updated Accession'}}, + }), + content_type='application/json', + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + models.Splocaleitemstr.objects.filter( + containername=self.container, language='en' + ).count(), + 1, + ) + string.refresh_from_db() + self.assertEqual(string.text, 'Updated Accession') + + def test_import_updates_existing_case_insensitive_country_string(self): + string = models.Splocaleitemstr.objects.create( + containername=self.container, + language='en', + country='US', + text='Existing US Accession', + ) + + response = self.client.post( + '/context/schema_localization_import.json', + data=json.dumps({ + 'language': 'en-US', + 'schema': {'accession': {'name': 'Updated US Accession'}}, + }), + content_type='application/json', + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + models.Splocaleitemstr.objects.filter( + containername=self.container, language='en' + ).count(), + 1, + ) + string.refresh_from_db() + self.assertEqual(string.text, 'Updated US Accession') + + def test_invalid_values_do_not_write(self): + response = self.client.post( + '/context/schema_localization_import.json', + data=json.dumps({ + 'schema': {'accession': {'isHidden': 'yes'}}, + 'language': 'en', + }), + content_type='application/json', + ) + + self.assertEqual(response.status_code, 400) + self.container.refresh_from_db() + self.assertIsNone(self.container.format) + + def test_rejects_non_schema_json(self): + response = self.client.post( + '/context/schema_localization_import.json', + data=json.dumps({'not': 'a schema'}), + content_type='application/json', + ) + + self.assertEqual(response.status_code, 400) + self.container.refresh_from_db() + self.assertIsNone(self.container.format) + + def test_rejects_invalid_language(self): + for language in ('en-us-extra', '@@', 'en-$%'): + with self.subTest(language=language): + response = self.client.post( + '/context/schema_localization_import.json', + data=json.dumps({ + 'language': language, + 'schema': {'accession': {'name': 'Should Not Import'}}, + }), + content_type='application/json', + ) + + self.assertEqual(response.status_code, 400) + self.assertFalse( + models.Splocaleitemstr.objects.filter( + containername=self.container, text='Should Not Import' + ).exists() + ) + + def test_rejects_non_object_data_for_known_table(self): + response = self.client.post( + '/context/schema_localization_import.json', + data=json.dumps({ + 'language': 'en', + 'schema': {'accession': 'invalid'}, + }), + content_type='application/json', + ) + + self.assertEqual(response.status_code, 400) diff --git a/specifyweb/backend/context/urls.py b/specifyweb/backend/context/urls.py index 22ad76b0008..844fcb0e7fd 100644 --- a/specifyweb/backend/context/urls.py +++ b/specifyweb/backend/context/urls.py @@ -2,7 +2,6 @@ Defines the urls for the app context subsystem """ -from django.urls import path from django.urls import path from . import views, user_resources, collection_resources @@ -30,6 +29,7 @@ path('viewsets.json', views.viewsets), path('datamodel.json', views.datamodel), path('schema_localization.json', views.schema_localization), + path('schema_localization_import.json', views.schema_localization_import), path('app.resource', views.app_resource), path('available_related_searches.json', views.available_related_searches), path('remoteprefs.properties', views.remote_prefs), diff --git a/specifyweb/backend/context/views.py b/specifyweb/backend/context/views.py index 4970bc6dd17..5524239b002 100644 --- a/specifyweb/backend/context/views.py +++ b/specifyweb/backend/context/views.py @@ -6,11 +6,13 @@ import os import re from typing import List +from xml.etree import ElementTree from django.conf import settings from django.contrib.auth import authenticate, login as auth_login, \ logout as auth_logout from django.db import connection, transaction +from django.db.models import Q from django.http import Http404, HttpResponse, HttpResponseBadRequest, \ HttpResponseForbidden, JsonResponse from django.urls import URLPattern @@ -28,7 +30,8 @@ check_permission_targets, skip_collection_access_check, query_pt, \ CollectionAccessPT from specifyweb.specify.models import Collection, Discipline, Division, Collectionobject, Institution, \ - Specifyuser, Spprincipal, Spversion, Collectionobjecttype + Specifyuser, Spprincipal, Spversion, Collectionobjecttype, Picklist, Splocalecontainer, \ + Splocalecontaineritem, Splocaleitemstr from specifyweb.specify.models_utils.schema import base_schema from specifyweb.specify.models_utils.serialize_datamodel import datamodel_to_json from specifyweb.specify.api.serializers import uri_for_model @@ -476,6 +479,190 @@ def schema_localization(request): lang = request.GET.get('lang', request.LANGUAGE_CODE) return JsonResponse(get_schema_localization(request.specify_collection, 0, lang)) + +SCHEMA_IMPORT_FIELDS = { + Splocalecontainer: {'format', 'aggregator', 'ishidden'}, + Splocalecontaineritem: { + 'format', 'ishidden', 'isrequired', 'picklistname', 'weblinkname', + }, +} +SCHEMA_IMPORT_BOOLEAN_FIELDS = {'ishidden', 'isrequired'} +SCHEMA_IMPORT_TABLE_KEYS = { + 'items', 'name', 'desc', *SCHEMA_IMPORT_FIELDS[Splocalecontainer] +} +SCHEMA_IMPORT_REFERENCE_FIELDS = {'format', 'picklistname', 'weblinkname'} + + +def _schema_import_resource_names(collection, user, resource, path): + result = get_app_resource(collection, user, resource) + if result is None: + return set() + try: + root = ElementTree.fromstring(result[0]) + except ElementTree.ParseError: + return set() + return { + (element.get('name') or element.text or '').lower() + for element in root.findall(path) + if element.get('name') or element.text + } + + +def _schema_import_values(data, fields, references): + if not isinstance(data, dict): + raise ValueError + values = {} + for key, value in data.items(): + key = key.lower() + if key not in fields: + continue + if key in SCHEMA_IMPORT_BOOLEAN_FIELDS: + if type(value) is not bool: + raise ValueError + elif value is not None and not isinstance(value, str): + raise ValueError + if key in SCHEMA_IMPORT_REFERENCE_FIELDS and value is not None \ + and value.lower() not in references[key]: + continue + values[key] = value + return values + + +def _schema_import_string(operations, parent, parent_field, text, language, country): + if text is None: + return + if not isinstance(text, str): + raise ValueError + country = country.lower() if country else None + country_filter = Q(country__isnull=True) | Q(country='') + if country is not None: + country_filter = Q(country__iexact=country) + string = Splocaleitemstr.objects.filter( + **{parent_field: parent, 'language': language} + ).filter(country_filter).filter( + Q(variant='') | Q(variant__isnull=True) + ).order_by('-id').first() + if string is None: + operations.append(( + 'POST', Splocaleitemstr, None, + { + 'text': text, + 'language': language, + 'country': country, + parent_field: uri_for_model(parent.__class__, parent.id), + }, + )) + elif string.text != text: + operations.append(('PUT', Splocaleitemstr, string, {'text': text})) + + +def _schema_import_operations(collection, schema, language, references=None): + if not isinstance(schema, dict) or not schema: + raise ValueError + if not any( + isinstance(data, dict) + and {key.lower() for key in data}.intersection(SCHEMA_IMPORT_TABLE_KEYS) + for data in schema.values() + ): + raise ValueError + language, _, country = language.lower().partition('-') + references = references or {key: set() for key in SCHEMA_IMPORT_REFERENCE_FIELDS} + containers = { + container.name.lower(): container + for container in Splocalecontainer.objects.filter( + discipline_id=collection.discipline_id, schematype=0 + ) + } + operations = [] + for table_name, table_data in schema.items(): + container = containers.get(table_name.lower()) + if container is None: + continue + if not isinstance(table_data, dict): + raise ValueError + values = _schema_import_values( + table_data, SCHEMA_IMPORT_FIELDS[Splocalecontainer], references + ) + if values: + operations.append(('PUT', Splocalecontainer, container, values)) + _schema_import_string( + operations, container, 'containername', table_data.get('name'), language, country or None + ) + _schema_import_string( + operations, container, 'containerdesc', table_data.get('desc'), language, country or None + ) + items = table_data.get('items', {}) + if not isinstance(items, dict): + raise ValueError + current_items = {item.name.lower(): item for item in container.items.all()} + for item_name, item_data in items.items(): + item = current_items.get(item_name.lower()) + if item is None: + continue + values = _schema_import_values( + item_data, SCHEMA_IMPORT_FIELDS[Splocalecontaineritem], references + ) + if values: + operations.append(('PUT', Splocalecontaineritem, item, values)) + _schema_import_string( + operations, item, 'itemname', item_data.get('name'), language, country or None + ) + _schema_import_string( + operations, item, 'itemdesc', item_data.get('desc'), language, country or None + ) + return operations + + +@login_maybe_required +@require_http_methods(['POST']) +def schema_localization_import(request): + try: + payload = json.loads(request.body) + schema = payload.get('schema', payload) + language = payload.get('language', request.LANGUAGE_CODE) + if not isinstance(language, str) or not re.fullmatch( + r'[A-Za-z]{2}(?:-[A-Za-z]{2})?', language + ): + raise ValueError + operations = _schema_import_operations( + request.specify_collection, + schema, + language, + { + 'format': _schema_import_resource_names( + request.specify_collection, request.specify_user, + 'DataObjFormatters', './/format' + ), + 'picklistname': { + name.lower() for name in Picklist.objects.filter( + collection=request.specify_collection + ).values_list('name', flat=True) + }, + 'weblinkname': _schema_import_resource_names( + request.specify_collection, request.specify_user, + 'WebLinks', './/weblinkdef/name' + ), + }, + ) + except (AttributeError, KeyError, TypeError, ValueError, json.JSONDecodeError): + return HttpResponseBadRequest() + + from specifyweb.specify.api.crud import post_resource, put_resource + + with transaction.atomic(): + for method, model, resource, data in operations: + if method == 'PUT': + put_resource( + request.specify_collection, request.specify_user_agent, + model.__name__, resource.id, resource.version, data + ) + else: + post_resource( + request.specify_collection, request.specify_user_agent, + model.__name__, data + ) + return JsonResponse({'updated': len(operations)}) + view_parameters_schema = [ { "name" : "name", diff --git a/specifyweb/frontend/js_src/lib/components/ChooseCollection/index.tsx b/specifyweb/frontend/js_src/lib/components/ChooseCollection/index.tsx index 53266302ad1..0545e2fb681 100644 --- a/specifyweb/frontend/js_src/lib/components/ChooseCollection/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/ChooseCollection/index.tsx @@ -172,7 +172,7 @@ function Wrapped({ loading( - ping('/accounts/logout/', {method: 'POST'}).then(() => + ping('/accounts/logout/', { method: 'POST' }).then(() => globalThis.location.assign( formatUrl('/specify/command/logout/', { next: nextUrl }) ) diff --git a/specifyweb/frontend/js_src/lib/components/Core/Main.tsx b/specifyweb/frontend/js_src/lib/components/Core/Main.tsx index 92a8971cc46..21c7bbb9fe4 100644 --- a/specifyweb/frontend/js_src/lib/components/Core/Main.tsx +++ b/specifyweb/frontend/js_src/lib/components/Core/Main.tsx @@ -102,7 +102,9 @@ function MissingAgent(): JSX.Element { }} forceToTop header={userText.noAgent()} - onClose={(): void => globalThis.location.assign('/specify/command/logout/')} + onClose={(): void => + globalThis.location.assign('/specify/command/logout/') + } > {userText.noAgentDescription()} diff --git a/specifyweb/frontend/js_src/lib/components/Router/Routes.tsx b/specifyweb/frontend/js_src/lib/components/Router/Routes.tsx index e5c97cfa656..1976bbc90e5 100644 --- a/specifyweb/frontend/js_src/lib/components/Router/Routes.tsx +++ b/specifyweb/frontend/js_src/lib/components/Router/Routes.tsx @@ -466,14 +466,11 @@ export const routes: RA = [ ({ CacheBuster }) => CacheBuster ), }, - { - path: 'logout', - title: userText.logOut(), - element: () => - import('../Logout').then( - ({ Logout }) => Logout - ) - }, + { + path: 'logout', + title: userText.logOut(), + element: () => import('../Logout').then(({ Logout }) => Logout), + }, ], }, { diff --git a/specifyweb/frontend/js_src/lib/components/SchemaConfig/Components.tsx b/specifyweb/frontend/js_src/lib/components/SchemaConfig/Components.tsx index 8adbbc64775..eb305e0ec73 100644 --- a/specifyweb/frontend/js_src/lib/components/SchemaConfig/Components.tsx +++ b/specifyweb/frontend/js_src/lib/components/SchemaConfig/Components.tsx @@ -18,12 +18,17 @@ export function SchemaConfigHeader({ languages, rawLanguage, onSave: handleSave, + onImport: handleImport, + importDisabled = false, }: { readonly languages: SchemaData['languages']; readonly rawLanguage: string; readonly onSave: (() => void) | undefined; + readonly onImport?: (file: File) => void; + readonly importDisabled?: boolean; }): JSX.Element { const [language] = rawLanguage.split('-'); + const importInput = React.useRef(null); return (

@@ -41,6 +46,28 @@ export function SchemaConfigHeader({ > {commonText.export()} + {handleImport !== undefined && ( + <> + { + const file = event.target.files?.[0]; + event.target.value = ''; + if (file !== undefined) handleImport(file); + }} + /> + importInput.current?.click()} + > + {commonText.import()} + + + )} {commonText.save()} diff --git a/specifyweb/frontend/js_src/lib/components/SchemaConfig/Layout.tsx b/specifyweb/frontend/js_src/lib/components/SchemaConfig/Layout.tsx index 48b9d165354..f70b220de9d 100644 --- a/specifyweb/frontend/js_src/lib/components/SchemaConfig/Layout.tsx +++ b/specifyweb/frontend/js_src/lib/components/SchemaConfig/Layout.tsx @@ -3,11 +3,18 @@ import { Outlet, useOutletContext } from 'react-router'; import { useMatch, useParams } from 'react-router-dom'; import { useUnloadProtect } from '../../hooks/navigation'; +import { commonText } from '../../localization/common'; import { schemaText } from '../../localization/schema'; +import { ajax } from '../../utils/ajax'; import { Container } from '../Atoms'; +import { Button } from '../Atoms/Button'; +import { Link } from '../Atoms/Link'; import { LoadingContext, ReadOnlyContext } from '../Core/Contexts'; +import { Dialog } from '../Molecules/Dialog'; +import { fileToText } from '../Molecules/FilePicker'; import { hasToolPermission } from '../Permissions/helpers'; import { SetSingleResourceContext } from '../Router/Router'; +import { formatUrl } from '../Router/queryString'; import { SchemaConfigHeader } from './Components'; import type { SchemaData } from './schemaData'; import { SchemaConfigSidebar } from './Sidebar'; @@ -16,6 +23,7 @@ import { SchemaConfigStoreProvider, useSchemaConfig, } from './Store'; +import { dialogIcons } from '../Atoms/Icons'; export function SchemaConfigLayout(): JSX.Element { const schemaData = useOutletContext(); @@ -46,6 +54,8 @@ function SchemaConfigLayoutContent(): JSX.Element { const tableName = match?.params.tableName ?? ''; const setSingleResource = React.useContext(SetSingleResourceContext); const loading = React.useContext(LoadingContext); + const [importFile, setImportFile] = React.useState(); + const [importError, setImportError] = React.useState(false); React.useEffect(() => { setSingleResource(`/specify/schema-config/${rawLanguage}/`); @@ -67,12 +77,37 @@ function SchemaConfigLayoutContent(): JSX.Element { }) ); }; + const handleImport = (file: File): void => { + setImportError(false); + setImportFile(file); + }; + const confirmImport = (): void => { + if (importFile === undefined) return; + const file = importFile; + setImportFile(undefined); + loading( + fileToText(file) + .then((text) => JSON.parse(text)) + .then((schema) => + ajax('/context/schema_localization_import.json', { + method: 'POST', + headers: { Accept: 'application/json' }, + body: { schema, language: rawLanguage }, + errorMode: 'silent', + }) + ) + .then(() => handleSchemaSaved(rawLanguage, tableName)) + .catch(() => setImportError(true)) + ); + }; return (
@@ -81,6 +116,64 @@ function SchemaConfigLayoutContent(): JSX.Element {
+ {importFile !== undefined && ( + + + {schemaText.downloadSchemaBackup({ + schemaConfig: schemaText.schemaConfig(), + })} + + + {commonText.cancel()} + + {schemaText.importSchemaContinue()} + + + } + icon={dialogIcons.warning} + header={schemaText.importSchema({ + schemaConfig: schemaText.schemaConfig(), + })} + onClose={(): void => setImportFile(undefined)} + > +

+ {schemaText.importSchemaWarning({ + schemaConfig: schemaText.schemaConfig(), + })} +

+

{schemaText.importSchemaLimitations()}

+

+ {schemaText.importSchemaBackupPrompt({ + schemaConfig: schemaText.schemaConfig(), + })} +

+ + )} + {importError && ( + {commonText.close()} + } + icon={dialogIcons.error} + header={schemaText.importSchema({ + schemaConfig: schemaText.schemaConfig(), + })} + onClose={(): void => setImportError(false)} + > +

+ {schemaText.importSchemaError({ + schemaConfig: schemaText.schemaConfig(), + })} +

+ + )} ); } diff --git a/specifyweb/frontend/js_src/lib/components/WbToolkit/GeoLocate.tsx b/specifyweb/frontend/js_src/lib/components/WbToolkit/GeoLocate.tsx index b4dfc1c845b..3feeb2a3ec9 100644 --- a/specifyweb/frontend/js_src/lib/components/WbToolkit/GeoLocate.tsx +++ b/specifyweb/frontend/js_src/lib/components/WbToolkit/GeoLocate.tsx @@ -12,9 +12,7 @@ import { filterArray } from '../../utils/types'; import { sortFunction } from '../../utils/utils'; import { Button } from '../Atoms/Button'; import { getLocalityField } from '../Leaflet/helpers'; -import { - getSelectedLocalityColumns, -} from '../Leaflet/wbLocalityDataExtractor'; +import { getSelectedLocalityColumns } from '../Leaflet/wbLocalityDataExtractor'; import type { GeoLocatePayload } from '../Molecules/GeoLocate'; import { GenericGeoLocate } from '../Molecules/GeoLocate'; import type { Dataset } from '../WbPlanView/Wrapped'; @@ -305,4 +303,4 @@ export function buildGeoLocateData( ) ) ); -} \ No newline at end of file +} diff --git a/specifyweb/frontend/js_src/lib/localization/schema.ts b/specifyweb/frontend/js_src/lib/localization/schema.ts index 59704141b98..cb33dddbedc 100644 --- a/specifyweb/frontend/js_src/lib/localization/schema.ts +++ b/specifyweb/frontend/js_src/lib/localization/schema.ts @@ -9,6 +9,31 @@ import { createDictionary } from './utils'; // Refer to "Guidelines for Programmers" in ./README.md before editing this file export const schemaText = createDictionary({ + importSchema: { + 'en-us': 'Import {schemaConfig:string}', + }, + importSchemaWarning: { + 'en-us': + 'Importing a {schemaConfig:string} will overwrite the current one. This action cannot be undone.', + }, + importSchemaBackupPrompt: { + 'en-us': + 'We strongly recommend downloading a backup of the current {schemaConfig:string} before importing.', + }, + importSchemaLimitations: { + 'en-us': + 'This import will not assign pick lists, field formats, or web links unless they already exist.', + }, + downloadSchemaBackup: { + 'en-us': 'Export {schemaConfig:string}', + }, + importSchemaContinue: { + 'en-us': 'Continue Import', + }, + importSchemaError: { + 'en-us': + 'The {schemaConfig:string} export provided is invalid and cannot be imported.', + }, table: { 'en-us': 'Table', 'ru-ru': 'Стол',