From 8c513ac7fe10ae2849a8c94c35a008acbcb4fd09 Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Sat, 14 Feb 2026 07:36:43 +0900 Subject: [PATCH 01/28] Add additional-funding array field, generalize WEKO deposit, and add workflow-run-id --- addons/weko/deposit.py | 20 +-- .../mappings/e-rad-metadata-mappings-csv.json | 49 ++++++ .../e-rad-metadata-mappings-ro-crate.json | 60 +++++++ addons/weko/schema/__init__.py | 3 +- addons/weko/schema/base.py | 83 ++++++--- addons/weko/schema/ro_crate_mebyo.py | 10 +- addons/weko/tests/test_schema.py | 130 ++++++++++++++ osf/migrations/0265_add_additional_funding.py | 28 +++ .../project/metadata/e-rad-metadata-1.json | 159 ++++++++++++++++++ 9 files changed, 495 insertions(+), 47 deletions(-) create mode 100644 osf/migrations/0265_add_additional_funding.py diff --git a/addons/weko/deposit.py b/addons/weko/deposit.py index 54226f825ae..153dcb3be4b 100644 --- a/addons/weko/deposit.py +++ b/addons/weko/deposit.py @@ -202,7 +202,6 @@ def _deposit_metadata( task_request_id=None, update_task_state=None, ): - from .schema.constants_mebyo import MEBYO_SCHEMA_NAME user = OSFUser.load(user_id) logger.info(f'Deposit: {metadata_paths}, {status_path} {task_request_id}') node = AbstractNode.load(node_id) @@ -328,8 +327,9 @@ def _deposit_metadata( logger.info(f'Uploading... {file_metadatas}') # 未病スキーマですでに WEKO 上にアイテムがある場合はバージョンアップ、それ以外の場合は新規作成 - if ro_crate_schemaname == MEBYO_SCHEMA_NAME and schema.get_weko_item_id(project_metadatas): - respbody = c.version_upgrade_item(schema.get_weko_item_id(project_metadatas), files, headers=headers) + weko_item_id = schema.get_weko_item_id(project_metadatas) if len(project_metadatas) == 1 else None + if weko_item_id: + respbody = c.version_upgrade_item(weko_item_id, files, headers=headers) else: respbody = c.deposit(files, headers=headers) logger.info(f'Uploaded: {respbody}') @@ -365,14 +365,14 @@ def _deposit_metadata( }, ) - if len(links) > 0 and ro_crate_schemaname == MEBYO_SCHEMA_NAME: + is_ingested = any( + s['@id'].endswith('/ingested') + for s in respbody.get('state', []) + if '@id' in s + ) + if is_ingested and len(project_metadatas) == 1: project_metadata = DraftRegistration.objects.filter(_id=metadata_node_id).first() - - weko_id = None - try: - weko_id = respbody['@id'].split('/')[-1] - except (KeyError, IndexError, TypeError): - pass + weko_id = respbody['@id'].split('/')[-1] if project_metadata and weko_id: update_data = { diff --git a/addons/weko/mappings/e-rad-metadata-mappings-csv.json b/addons/weko/mappings/e-rad-metadata-mappings-csv.json index 176dec3f890..e6be93f55f0 100644 --- a/addons/weko/mappings/e-rad-metadata-mappings-csv.json +++ b/addons/weko/mappings/e-rad-metadata-mappings-csv.json @@ -77,6 +77,55 @@ } } }, + "additional-funding": { + "@type": "jsonarray", + "metadata.item_30002_funding_reference21[]": { + "subitem_funder_identifiers": { + "@createIf": "{{ object_funder }}", + "subitem_funder_identifier_type": "e-Rad_funder" + }, + "subitem_funder_names[FUNDER_NAME_JA]": { + "@createIf": "{{ object_funder }}", + "subitem_funder_name": "{{object_funder_tooltip_0}}({{object_funder}})", + "subitem_funder_name_language": "ja" + }, + "subitem_funder_names[FUNDER_NAME_EN]": { + "@createIf": "{{ object_funder }}", + "subitem_funder_name": "{{object_funder_tooltip_1}}({{object_funder}})", + "subitem_funder_name_language": "en" + }, + "subitem_funding_stream_identifiers": { + "@createIf": "{{ object_funding_stream_code }}", + "subitem_funding_stream_identifier": "{{object_funding_stream_code}}", + "subitem_funding_stream_identifier_type": "JGN_fundingStream" + }, + "subitem_funding_streams[FUNDING_STREAM_JA]": { + "@createIf": "{{ object_program_name_ja }}", + "subitem_funding_stream": "{{object_program_name_ja}}", + "subitem_funding_stream_language": "ja" + }, + "subitem_funding_streams[FUNDING_STREAM_EN]": { + "@createIf": "{{ object_program_name_en }}", + "subitem_funding_stream": "{{object_program_name_en}}", + "subitem_funding_stream_language": "en" + }, + "subitem_award_numbers": { + "@createIf": "{{ object_japan_grant_number }}", + "subitem_award_number_type": "JGN", + "subitem_award_number": "{{object_japan_grant_number}}" + }, + "subitem_award_titles[PROJECT_NAME_JA]": { + "@createIf": "{{ object_project_name_ja }}", + "subitem_award_title_language": "ja", + "subitem_award_title": "{{object_project_name_ja}}" + }, + "subitem_award_titles[PROJECT_NAME_EN]": { + "@createIf": "{{ object_project_name_en }}", + "subitem_award_title_language": "en", + "subitem_award_title": "{{object_project_name_en}}" + } + } + }, "grdm-file:data-number": { "@type": "string", "@createIf": "{% if grdm_file_file_type_value != \"manuscript\" %}{{value}}{% endif %}" diff --git a/addons/weko/mappings/e-rad-metadata-mappings-ro-crate.json b/addons/weko/mappings/e-rad-metadata-mappings-ro-crate.json index 98fcee87a46..c50f602cce4 100644 --- a/addons/weko/mappings/e-rad-metadata-mappings-ro-crate.json +++ b/addons/weko/mappings/e-rad-metadata-mappings-ro-crate.json @@ -80,6 +80,66 @@ } } }, + "additional-funding": { + "@type": "jsonarray", + "root.jpcoar:fundingReference[]": { + ".@type": "PropertyValue", + "jpcoar:funderIdentifier": { + "@createIf": "{{ object_funder | to_funder_ror_id }}", + ".@type": "jpcoar:funderIdentifier", + "jpcoar:funderIdentifierType": "ROR", + "value": "{{ object_funder | to_funder_ror_id }}" + }, + "jpcoar:funderName[FUNDER_NAME_JA]": { + "@createIf": "{{ object_funder }}", + ".@type": "PropertyValue", + "value": "{{object_funder_tooltip_0}}({{object_funder}})", + "language": "ja" + }, + "jpcoar:funderName[FUNDER_NAME_EN]": { + "@createIf": "{{ object_funder }}", + ".@type": "PropertyValue", + "value": "{{object_funder_tooltip_1}}({{object_funder}})", + "language": "en" + }, + "jpcoar:fundingStreamIdentifier": { + "@createIf": "{{ object_funding_stream_code }}", + ".@type": "jpcoar:fundingStreamIdentifier", + "value": "{{object_funding_stream_code}}", + "jpcoar:fundingStreamIdentifierType": "JGN_fundingStream" + }, + "jpcoar:fundingStream[FUNDING_STREAM_JA]": { + "@createIf": "{{ object_program_name_ja }}", + ".@type": "PropertyValue", + "value": "{{object_program_name_ja}}", + "language": "ja" + }, + "jpcoar:fundingStream[FUNDING_STREAM_EN]": { + "@createIf": "{{ object_program_name_en }}", + ".@type": "PropertyValue", + "value": "{{object_program_name_en}}", + "language": "en" + }, + "jpcoar:awardNumber": { + "@createIf": "{{ object_japan_grant_number }}", + ".@type": "jpcoar:awardNumber", + "jpcoar:awardNumberType": "JGN", + "value": "{{object_japan_grant_number}}" + }, + "jpcoar:awardTitle[PROJECT_NAME_JA]": { + "@createIf": "{{ object_project_name_ja }}", + ".@type": "PropertyValue", + "language": "ja", + "value": "{{object_project_name_ja}}" + }, + "jpcoar:awardTitle[PROJECT_NAME_EN]": { + "@createIf": "{{ object_project_name_en }}", + ".@type": "PropertyValue", + "language": "en", + "value": "{{object_project_name_en}}" + } + } + }, "grdm-file:data-number": { "@type": "string", "@createIf": "{% if grdm_file_file_type_value != \"manuscript\" %}{{value}}{% endif %}" diff --git a/addons/weko/schema/__init__.py b/addons/weko/schema/__init__.py index e61ba4d9734..85a7abcf7b5 100644 --- a/addons/weko/schema/__init__.py +++ b/addons/weko/schema/__init__.py @@ -1,7 +1,6 @@ -from .base import get_available_schema_id +from .base import get_available_schema_id, get_weko_item_id from .csv import write_csv from .ro_crate import write_ro_crate_json -from .ro_crate_mebyo import get_weko_item_id __all__ = [ 'get_available_schema_id', diff --git a/addons/weko/schema/base.py b/addons/weko/schema/base.py index 1d9860b0c9f..e9eb8ac3b57 100644 --- a/addons/weko/schema/base.py +++ b/addons/weko/schema/base.py @@ -2,6 +2,7 @@ import json import logging import re +from typing import Any, Union from jinja2 import Environment @@ -11,6 +12,19 @@ logger = logging.getLogger(__name__) +def get_weko_item_id(project_metadatas: Any) -> Union[str, None]: + '''プロジェクトメタデータから JAIRO Cloud の item_id を取得''' + if len(project_metadatas) != 1: + raise ValueError('Choose 1 project metadata to export.') + project_metadata = project_metadatas[0] + if isinstance(project_metadata, str): + project_metadata = json.loads(project_metadata) + item_id = project_metadata.get('internal:weko-item-id') + if item_id and item_id.get('value'): + return item_id.get('value') + return None + + def _get_metadata_value(file_metadata_data, item, lang, index): assert 'type' in item, item if item['type'] == 'const': @@ -32,6 +46,34 @@ def _get_metadata_value(file_metadata_data, item, lang, index): return json.loads(value)[index][item['value']] raise KeyError(item['type']) +def _resolve_options(value, options): + normalized = [] + for o in options: + if isinstance(o, str): + normalized.append({'text': o}) + elif isinstance(o, dict): + normalized.append(o) + else: + logger.debug(f'Unexpected option type: {type(o)} for value={value}') + filtered = [ + o for o in normalized + if o.get('text', None) == value or (not value and o.get('default', False)) + ] + if not filtered: + return None, {} + option = filtered[0] + tooltips = {} + if 'tooltip' in option: + tooltips['tooltip'] = option['tooltip'] + langs = option['tooltip'].split('|') + if len(langs) > 1: + for i, s in enumerate(langs): + tooltips[f'tooltip_{i}'] = s + else: + for i in range(2): + tooltips[f'tooltip_{i}'] = option['tooltip'] + return option['text'], tooltips + def _get_item_variables(file_metadata, schema=None): values = { 'value': '', @@ -41,47 +83,34 @@ def _get_item_variables(file_metadata, schema=None): if 'value' in file_metadata: v = file_metadata['value'] if schema is not None and 'options' in schema: - options = [] - for o in schema['options']: - if isinstance(o, str): - options.append({'text': o}) - elif isinstance(o, dict): - options.append(o) - else: - logger.debug(f'Unexpected option type: {type(o)} for value={v}') - filtered_options = [ - o - for o in options - if o.get('text', None) == v or (not v and o.get('default', False)) - ] - if len(filtered_options) == 0: + resolved, tooltips = _resolve_options(v, schema['options']) + if resolved is None: logger.debug(f'No suitable options: value={v}, schema={schema}') else: - option = filtered_options[0] - v = option['text'] - if 'tooltip' in option: - values['tooltip'] = option['tooltip'] - langs = option['tooltip'].split('|') - if len(langs) > 1: - for i, s in enumerate(langs): - values[f'tooltip_{i}'] = s - else: - for i in range(2): - values[f'tooltip_{i}'] = option['tooltip'] + v = resolved + values.update(tooltips) values['value'] = v if 'object' in file_metadata: o = file_metadata['object'] - values.update(_get_object_variables(o, 'object_')) + values.update(_get_object_variables(o, 'object_', schema=schema)) return values -def _get_object_variables(o, prefix): +def _get_object_variables(o, prefix, schema=None): values = {} + properties = schema['properties'] if schema is not None and 'properties' in schema else None for k, v in o.items(): key_ = k.replace('-', '_').replace(':', '_').replace('.', '_') if isinstance(v, dict): values.update(_get_object_variables(v, f'{prefix}{key_}_')) continue values[f'{prefix}{key_}'] = v + if properties is not None: + prop_schema = next((p for p in properties if p['id'] == k), None) + if prop_schema is not None and 'options' in prop_schema: + resolved, tooltips = _resolve_options(v, prop_schema['options']) + if resolved is not None: + for tk, tv in tooltips.items(): + values[f'{prefix}{key_}_{tk}'] = tv return values def get_value(file_metadata, text, commonvars=None, schema=None): diff --git a/addons/weko/schema/ro_crate_mebyo.py b/addons/weko/schema/ro_crate_mebyo.py index 22432411d65..09c8b948cdf 100644 --- a/addons/weko/schema/ro_crate_mebyo.py +++ b/addons/weko/schema/ro_crate_mebyo.py @@ -184,11 +184,5 @@ def generate_dataset_metadata(project_metadatas: Any) -> Tuple[List[Dict[str, An def get_weko_item_id(project_metadatas: Any) -> Union[str, None]: '''未病スキーマのプロジェクトメタデータから JAIRO Cloud の item_id を取得''' - if len(project_metadatas) != 1: - raise ValueError('Choose 1 project metadata to export.') - project_metadata = _deep_json_loads(project_metadatas[0]) - item_id = project_metadata.get('internal:weko-item-id') - - if item_id and item_id.get('value'): - return item_id.get('value') - return None + from .base import get_weko_item_id as _get_weko_item_id + return _get_weko_item_id(project_metadatas) diff --git a/addons/weko/tests/test_schema.py b/addons/weko/tests/test_schema.py index bb71f207bea..6111a72a0be 100644 --- a/addons/weko/tests/test_schema.py +++ b/addons/weko/tests/test_schema.py @@ -2081,3 +2081,133 @@ def test_write_ro_crate_json_erad_requires_files(self): str(context.exception), 'Error message should indicate missing file metadata' ) + + def test_write_ro_crate_json_additional_funding(self): + buf = io.StringIO() + index = mock.MagicMock() + index.identifier = '1000' + index.title = 'TITLE' + node_id = 'rvm3q' + files = [ + [('test.jpg', 'image/jpeg')], + ] + target_schema = RegistrationSchema.objects \ + .filter(name='公的資金による研究データのメタデータ登録') \ + .order_by('-schema_version') \ + .first() + file_metadata = { + 'items': [ + { + 'schema': target_schema._id, + 'data': dict([(k, { + 'value': v, + })for k, v in { + 'grdm-file:title-en': 'TEST DATA', + 'grdm-file:data-description-ja': 'テスト説明', + }.items()]), + }, + ], + } + project_metadata = { + 'funder': { + 'value': 'JST', + }, + 'japan-grant-number': { + 'value': 'JP100001', + }, + 'project-name-ja': { + 'value': 'メインプロジェクト', + }, + 'project-name-en': { + 'value': 'Main Project', + }, + 'additional-funding': { + 'value': [ + { + 'funder': 'JSPS', + 'japan-grant-number': 'JP200002', + 'project-name-ja': '追加プロジェクト1', + 'project-name-en': 'Additional Project 1', + }, + { + 'funder': 'AMED', + 'japan-grant-number': 'JP300003', + 'project-name-ja': '追加プロジェクト2', + 'project-name-en': 'Additional Project 2', + }, + ], + }, + } + + schema.write_ro_crate_json( + self.user, + buf, + index, + files, + target_schema._id, + [file_metadata], + [project_metadata], + node_id + ) + + actual_json = json.loads(buf.getvalue()) + graph = actual_json['@graph'] + entities = {e['@id']: e for e in graph if '@id' in e} + + def resolve_ref(ref): + return entities[ref['@id']] + + dataset = entities['./'] + funding_refs = dataset['jpcoar:fundingReference'] + assert_equal(len(funding_refs), 3) + + resolved_fundings = [] + for ref in funding_refs: + fr = resolve_ref(ref) + assert_equal(fr['@type'], 'PropertyValue') + + award_number = resolve_ref(fr['jpcoar:awardNumber']) + assert_equal(award_number['@type'], 'jpcoar:awardNumber') + assert_equal(award_number['jpcoar:awardNumberType'], 'JGN') + + funder_names = [resolve_ref(r) for r in fr['jpcoar:funderName']] + assert_equal(len(funder_names), 2) + funder_ja = next(n for n in funder_names if n['language'] == 'ja') + funder_en = next(n for n in funder_names if n['language'] == 'en') + + award_titles = [resolve_ref(r) for r in fr['jpcoar:awardTitle']] + assert_equal(len(award_titles), 2) + title_ja = next(t for t in award_titles if t['language'] == 'ja') + title_en = next(t for t in award_titles if t['language'] == 'en') + + resolved_fundings.append({ + 'grant_number': award_number['value'], + 'funder_ja': funder_ja['value'], + 'funder_en': funder_en['value'], + 'title_ja': title_ja['value'], + 'title_en': title_en['value'], + }) + + resolved_fundings.sort(key=lambda f: f['grant_number']) + + assert_equal(resolved_fundings[0], { + 'grant_number': 'JP100001', + 'funder_ja': '国立研究開発法人科学技術振興機構(JST)', + 'funder_en': 'Japan Science and Technology Agency(JST)', + 'title_ja': 'メインプロジェクト', + 'title_en': 'Main Project', + }) + assert_equal(resolved_fundings[1], { + 'grant_number': 'JP200002', + 'funder_ja': '独立行政法人日本学術振興会(JSPS)', + 'funder_en': 'Japan Society for the Promotion of Science(JSPS)', + 'title_ja': '追加プロジェクト1', + 'title_en': 'Additional Project 1', + }) + assert_equal(resolved_fundings[2], { + 'grant_number': 'JP300003', + 'funder_ja': '国立研究開発法人日本医療研究開発機構(AMED)', + 'funder_en': 'Japan Agency for Medical Research and Development(AMED)', + 'title_ja': '追加プロジェクト2', + 'title_en': 'Additional Project 2', + }) diff --git a/osf/migrations/0265_add_additional_funding.py b/osf/migrations/0265_add_additional_funding.py new file mode 100644 index 00000000000..7075c3d14be --- /dev/null +++ b/osf/migrations/0265_add_additional_funding.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +from osf.utils.migrations import UpdateRegistrationSchemasAndSchemaBlocks + + +def ensure_registration_mappings(*args): + from api.base import settings + from addons.weko.apps import NAME + from addons.weko.utils import ensure_registration_metadata_mapping + from addons.weko.mappings import REGISTRATION_METADATA_MAPPINGS + if NAME not in settings.INSTALLED_APPS: + return + for schema_name, mappings in REGISTRATION_METADATA_MAPPINGS: + ensure_registration_metadata_mapping(schema_name, mappings) + + +class Migration(migrations.Migration): + + dependencies = [ + ('osf', '0264_merge_20260208_0831'), + ] + + operations = [ + UpdateRegistrationSchemasAndSchemaBlocks(), + migrations.RunPython(ensure_registration_mappings, ensure_registration_mappings), + ] diff --git a/website/project/metadata/e-rad-metadata-1.json b/website/project/metadata/e-rad-metadata-1.json index b193e68036e..b72b95b66ed 100644 --- a/website/project/metadata/e-rad-metadata-1.json +++ b/website/project/metadata/e-rad-metadata-1.json @@ -211,6 +211,143 @@ } ], "required": true + }, + { + "qid": "additional-funding", + "nav": "追加の助成情報", + "title": "追加の助成情報|Additional Funding", + "help": "追加の資金配分機関情報がある場合は入力してください。|Enter additional funding information if applicable.", + "type": "array", + "properties": [ + { + "id": "japan-grant-number", + "title": "体系的番号|Japan Grant Number", + "type": "string", + "format": "japan-grant-number" + }, + { + "id": "funder", + "title": "資金配分機関情報|Funder", + "type": "choose", + "format": "e-rad-award-funder", + "options": [ + { + "text": "AMED", + "tooltip": "国立研究開発法人日本医療研究開発機構|Japan Agency for Medical Research and Development|303" + }, + { + "text": "MOD", + "tooltip": "防衛省|Ministry of Defense|501" + }, + { + "text": "NRA", + "tooltip": "原子力規制庁|Nuclear Regulation Authority|1503" + }, + { + "text": "ERCA", + "tooltip": "独立行政法人環境再生保全機構|Environmental Restoration and Conservation Agency|1502" + }, + { + "text": "CAO", + "tooltip": "内閣府|Cabinet Office|301" + }, + { + "text": "SOUMU", + "tooltip": "総務省|Ministry of Internal Affairs and Communications|601" + }, + { + "text": "FDMA", + "tooltip": "消防庁|Fire and Disaster Management Agency|605" + }, + { + "text": "NICT", + "tooltip": "国立研究開発法人情報通信研究機構|National Institute of Information and Communications Technology|615" + }, + { + "text": "MEXT", + "tooltip": "文部科学省|Ministry of Education|1001" + }, + { + "text": "JST", + "tooltip": "国立研究開発法人科学技術振興機構|Japan Science and Technology Agency|1020" + }, + { + "text": "JSPS", + "tooltip": "独立行政法人日本学術振興会|Japan Society for the Promotion of Science|1025" + }, + { + "text": "MHLW", + "tooltip": "厚生労働省|Ministry of Health|1101" + }, + { + "text": "MAFF", + "tooltip": "農林水産省|Ministry of Agriculture|1201" + }, + { + "text": "PRIMAFF", + "tooltip": "農林水産省農林水産政策研究所|Policy Research Institute|1203" + }, + { + "text": "NARO", + "tooltip": "国立研究開発法人農業・食品産業技術総合研究機構|National Agriculture and Food Research Organization|1205" + }, + { + "text": "METI", + "tooltip": "経済産業省|Ministry of Economy|1301" + }, + { + "text": "NEDO", + "tooltip": "国立研究開発法人新エネルギー・産業技術総合開発機構|New Energy and Industrial Technology Development|1305 Organization" + }, + { + "text": "MLIT", + "tooltip": "国土交通省|Ministry of Land|1401" + }, + { + "text": "NILIM", + "tooltip": "国土技術政策総合研究所|National Institute for Land and Infrastructure Management|1405" + }, + { + "text": "ENV", + "tooltip": "環境省|Ministry of the Environment|1501" + }, + { + "text": "BIBIOHN", + "tooltip": "国立研究開発法人医薬基盤・健康・栄養研究所|National Institutes of Biomedical Innovation|1106" + } + ] + }, + { + "id": "funding-stream-code", + "title": "体系的番号におけるプログラム情報コード|Funding stream code", + "type": "string", + "format": "funding-stream-code" + }, + { + "id": "program-name-ja", + "title": "プログラム名 (日本語)|Program name (Japanese)", + "type": "string", + "format": "jgn-program-name-ja" + }, + { + "id": "program-name-en", + "title": "Program name (English)", + "type": "string", + "format": "jgn-program-name-en" + }, + { + "id": "project-name-ja", + "title": "プロジェクト名 (日本語)|Project name (Japanese)", + "type": "string", + "format": "e-rad-award-title-ja" + }, + { + "id": "project-name-en", + "title": "Project name (English)", + "type": "string", + "format": "e-rad-award-title-en" + } + ] } ] }, @@ -2065,6 +2202,28 @@ } } ] + }, + { + "id": "page3", + "title": "内部用編集不可メタデータ|Uneditable Internal Metadata", + "hide_projectmetadata": true, + "concealment_page_navigator": true, + "description": "プロジェクトメタデータ・ファイルメタデータのどちらにも表示せずユーザー編集不可とする内部用メタデータ", + "questions": [ + { + "qid": "internal:weko-item-id", + "type": "string", + "format": "text", + "pattern": "^[0-9]+$", + "required": false + }, + { + "qid": "internal:workflow-run-id", + "type": "string", + "format": "text", + "required": false + } + ] } ] } \ No newline at end of file From e7b657057b402c2406d433da39b201c16573b08a Mon Sep 17 00:00:00 2001 From: yacchin1205 <968739+yacchin1205@users.noreply.github.com> Date: Sat, 7 Mar 2026 07:21:30 +0900 Subject: [PATCH 02/28] Add metadata UI enhancements: schema ui hints, name field split, and suggestion autofill --- addons/metadata/README.md | 82 + addons/metadata/routes.py | 2 + addons/metadata/static/files.js | 51 +- addons/metadata/static/metadata-fields.js | 547 ++++++- addons/metadata/suggestions/erad.py | 74 +- addons/metadata/tests/test_suggestion.py | 37 + addons/metadata/tests/test_utils.py | 124 ++ addons/metadata/utils.py | 174 +++ .../mappings/e-rad-metadata-mappings-csv.json | 83 +- .../e-rad-metadata-mappings-ro-crate.json | 87 +- addons/weko/schema/ro_crate.py | 3 + .../scripts/example-manuscript-metadata.json | 8 +- addons/weko/scripts/example-metadata.json | 8 +- addons/weko/tests/test_schema.py | 447 +++++- api/schemas/serializers.py | 1 + ...266_add_ui_to_registration_schema_block.py | 24 + osf/migrations/0267_split_name_fields.py | 46 + osf/models/metaschema.py | 1 + osf/utils/migrations.py | 6 +- .../project/metadata/e-rad-metadata-1.json | 1318 ++++++++++++----- .../en/LC_MESSAGES/js_messages.po | 2 +- .../ja/LC_MESSAGES/js_messages.po | 2 +- 22 files changed, 2554 insertions(+), 573 deletions(-) create mode 100644 osf/migrations/0266_add_ui_to_registration_schema_block.py create mode 100644 osf/migrations/0267_split_name_fields.py diff --git a/addons/metadata/README.md b/addons/metadata/README.md index ee8744e98c2..d1a92cfe052 100644 --- a/addons/metadata/README.md +++ b/addons/metadata/README.md @@ -34,6 +34,88 @@ USE_DATASET_IMPORTING = True The "Import Dataset" button is displayed in a toolbar of a file browser if `USE_DATASET_IMPORTING` is true and users can import datasets from external sources. +## Schema UI Hints (`ui` property) + +Metadata schemas (e.g. `e-rad-metadata-1.json`) define both data structure and form rendering. To keep rendering concerns separate from data structure, each question may carry an optional `ui` property — a single JSON object that holds all UI-specific hints. + +### Design rationale + +Schema properties flow through two distinct paths: + +1. **File metadata path** — The schema JSON is stored in `RegistrationSchema.schema` (a JSONField) and served as-is to the browser. The frontend JS (`metadata-fields.js`) reads `question.ui.*` directly. Any new key added to `ui` is automatically available without backend changes. + +2. **Project metadata path** — The migration `map_schemas_to_schemablocks` decomposes each question into `RegistrationSchemaBlock` rows. Each column on that model requires a DB migration, a serializer update, and a corresponding Ember model attribute. Adding a single column touches Python models, migrations, API serializers, and Ember types. + +By consolidating all UI hints into a single `ui` JSONField column on `RegistrationSchemaBlock`, the project metadata path gains the same extensibility as the file metadata path: new hints can be added to the `ui` object without further migrations or model changes. + +### Structure + +Simple ja/en pair: + +```json +{ "qid": "grdm-file:title-ja", + "ui": { + "group": { "id": "title", "title": "データの名称|Title", "tags": ["共通|Common", "リポジトリ|Repository"], + "help": "...", "info": "管理対象データの特徴を示す名称を入力。|..." }, + "sub_label": "（日本語）|(Japanese)", + "item": { "placeholder": "研究データ管理に関する意識調査" } + } } + +{ "qid": "grdm-file:title-en", + "ui": { "group": "title", "sub_label": "（English）|(English)" } } +``` + +Nested group (ja/en pair inside a category): + +```json +{ "qid": "grdm-file:data-policy-free", + "ui": { "group": { "id": "data-policy", "title": "管理対象データの利活用・提供方針|...", "bar": true, + "tags": ["共通|Common"], "info": "ライセンス情報を記載。|..." } } } + +{ "qid": "grdm-file:data-policy-license", + "ui": { "group": "data-policy" } } + +{ "qid": "grdm-file:data-policy-cite-ja", + "ui": { + "group": { "id": "data-policy-cite", "parent": "data-policy", "title": "引用方法等|..." }, + "sub_label": "（日本語）|(Japanese)" + } } + +{ "qid": "grdm-file:data-policy-cite-en", + "ui": { "group": "data-policy-cite", "sub_label": "（English）|(English)" } } +``` + +The `ui` object has three scopes: + +**Page scope** (`page.ui`) — page-level decoration: + +- **`header`** — Introductory HTML text displayed above all questions on the page (e.g. format notes, tag legend). + +**Group scope** (`ui.group`) — section heading and grouping: + +- **`id`** — Group identifier. The first question in a group carries the full definition (object); subsequent questions reference the group by `id` string only (e.g. `"group": "title"`). +- **`parent`** — Parent group reference. A string ID when the parent is already defined by an earlier question. An object (`{ "id": "...", "title": "...", ... }`) to simultaneously define the parent group — useful when no earlier question belongs directly to the parent. +- **`title`** — Section heading text. +- **`help`** — Help text (supports HTML) displayed under the section heading. +- **`bar`** — If `true`, draw a continuous vertical bar on the left side of group members (for semantic category groups). +- **`tags`** — Badge labels for the section (e.g. `["共通|Common"]`, `["共通|Common", "リポジトリ|Repository"]`). Pipe-delimited for localization. +- **`info`** — Detailed explanation shown in a popover when the ⓘ mark next to the group heading is clicked. + +**Item scope** (`ui.item`) — individual input field: + +- **`placeholder`** — Placeholder text for the input field. +- **`width`** — Abstract width category (e.g. `"narrow"`, `"half"`). +- **`widget`** — Override the default widget (e.g. `"radio"` instead of pulldown for `singleselect`). +- **`enabled_if`** — Conditional disabled display. An object with a `disabled` key whose value is either `true` (unconditionally disabled) or a condition object (e.g. `{ "disabled": { "grdm-file:file-type": "dataset" } }`). When the question's `enabled_if` is false and the `disabled` condition is met, the field is shown greyed out instead of hidden. +- **`info`** — Detailed explanation shown in a popover when the ⓘ mark next to the field label is clicked. For grouped fields, prefer `ui.group.info` over `ui.item.info`. +- **`tags`** — Badge labels for standalone fields (fields not belonging to a group). Pipe-delimited for localization. + +**Top-level** (`ui.*`): + +- **`sub_label`** — Label within a group (e.g. "（日本語）|(Japanese)" / "（English）|(English)" for ja/en pairs). + +This list is not exhaustive; new keys can be added as needed without schema migration. + ## Suggestion Policies (ERAD/KAKEN) This addon provides researcher/project suggestions sourced from ERAD and KAKEN. Ordering and deduplication follow simple, explicit policies so results are predictable and easy to reason about. diff --git a/addons/metadata/routes.py b/addons/metadata/routes.py index ad519dcb459..e18d9c416df 100644 --- a/addons/metadata/routes.py +++ b/addons/metadata/routes.py @@ -61,6 +61,8 @@ Rule([ '/project//{}/file_metadata/suggestions/files/'.format(SHORT_NAME), '/project//node//{}/file_metadata/suggestions/files/'.format(SHORT_NAME), + '/project//{}/suggestions'.format(SHORT_NAME), + '/project//node//{}/suggestions'.format(SHORT_NAME), ], 'get', views.metadata_file_metadata_suggestions, json_renderer), Rule([ '/{}/packages/projects/'.format(SHORT_NAME), diff --git a/addons/metadata/static/files.js b/addons/metadata/static/files.js index efc8f2422f8..1d252850d6a 100644 --- a/addons/metadata/static/files.js +++ b/addons/metadata/static/files.js @@ -13,6 +13,7 @@ const _ = rdmGettext._; const ImportDatasetButton = require('./metadataImportDatasetButton.js'); const QuestionPage = require('./metadata-fields.js').QuestionPage; +const getLocalizedText = require('./util.js').getLocalizedText; const WaterButlerCache = require('./wbcache.js').WaterButlerCache; const registrations = require('./registration.js'); const RegistrationSchemas = registrations.RegistrationSchemas; @@ -293,9 +294,7 @@ function MetadataButtons() { ); self.lastFields = self.lastQuestionPage.fields; container.empty(); - self.lastFields.forEach(function(field) { - container.append(field.element); - }); + container.append(self.lastQuestionPage.container); self.lastQuestionPage.validateAll(); } @@ -334,8 +333,9 @@ function MetadataButtons() { } self.createSchemaSelector = function(targetItem) { - const label = $('').text(_('Metadata Schema:')); - const schema = $(''); + const label = $('').text(_('Schema:')) + .css({ 'margin-right': '8px', 'margin-bottom': 0, 'white-space': 'nowrap', 'min-width': '8em', 'text-align': 'right' }); + const schema = $('').addClass('form-control'); const activeSchemas = (self.registrationSchemas.schemas || []) .filter(function(s) { return s.attributes.active; @@ -346,7 +346,7 @@ function MetadataButtons() { activeSchemas.forEach(function(s) { schema.append($('') .attr('value', s.id) - .text(s.attributes.name)); + .text(getLocalizedText(s.attributes.schema.ui && s.attributes.schema.ui.label) || s.attributes.name)); }); var currentSchemaId = null; const activeSchemaIds = activeSchemas.map(function(s) { @@ -363,6 +363,7 @@ function MetadataButtons() { schema.val(currentSchemaId); } const group = $('
').addClass('form-group') + .css({ 'margin-bottom': 0, display: 'flex', 'align-items': 'center' }) .append(label) .append(schema); return { @@ -521,10 +522,12 @@ function MetadataButtons() { self.findSchemaById(self.currentSchemaId), filepath, item, - {} + { variantContainer: variantSlot } ); }); - dialog.toolbar.append(selector.group); + var variantSlot = $('
'); + var selectorColumn = $('
').append(selector.group).append(variantSlot); + dialog.toolbar.append(selectorColumn); if ((context.projectMetadata || {}).editable && !extraMetadata) { const pasteButton = $('') .addClass('btn btn-default') @@ -534,9 +537,7 @@ function MetadataButtons() { .append(_('Paste from Clipboard')) .attr('type', 'button') .on('click', self.pasteFromClipboard); - dialog.toolbar.append($('
') - .css('display', 'flex') - .append(pasteButton)); + dialog.toolbar.append(pasteButton); } if (dialog.customHandler) { dialog.customHandler.empty(); @@ -566,7 +567,7 @@ function MetadataButtons() { self.findSchemaById(self.currentSchemaId), filepath, item, - {} + { variantContainer: variantSlot } ); dialog.container.append(fieldContainer); dialog.dialog.one('shown.bs.modal', function() { @@ -656,11 +657,13 @@ function MetadataButtons() { self.findSchemaById(self.currentSchemaId), filepaths, items, - Object.assign({multiple: true}, computeValuesForMultipleEdit(self.currentSchemaId)) + Object.assign({multiple: true, variantContainer: variantSlot}, computeValuesForMultipleEdit(self.currentSchemaId)) ); }); dialog.toolbar.empty(); - dialog.toolbar.append(selector.group); + var variantSlot = $('
'); + var selectorColumn = $('
').append(selector.group).append(variantSlot); + dialog.toolbar.append(selectorColumn); // container dialog.container.empty(); @@ -671,7 +674,7 @@ function MetadataButtons() { self.findSchemaById(self.currentSchemaId), filepaths, items, - Object.assign({multiple: true}, computeValuesForMultipleEdit(self.currentSchemaId)) + Object.assign({multiple: true, variantContainer: variantSlot}, computeValuesForMultipleEdit(self.currentSchemaId)) ); dialog.container.append(fieldContainer); dialog.dialog.one('shown.bs.modal', function() { @@ -1131,9 +1134,7 @@ function MetadataButtons() { ); self.lastFields = self.lastQuestionPage.fields; container.empty(); - self.lastFields.forEach(function(field) { - container.append(field.element); - }); + container.append(self.lastQuestionPage.container); self.lastQuestionPage.validateAll(); const message = $('
'); if (self.lastQuestionPage.hasValidationError) { @@ -2208,7 +2209,10 @@ function MetadataButtons() { copyToClipboard.on('click', function(event) { self.copyToClipboard(event, copyStatus); }); - const toolbar = $('
'); + const toolbar = $('
') + .css('display', 'flex') + .css('align-items', 'flex-end') + .css('margin-bottom', '10px'); const customHandler = $(''); const container = $('
    ').css('padding', '0 20px'); var notice = $(''); @@ -2232,6 +2236,8 @@ function MetadataButtons() { .append($('
    ') .css('overflow-y', 'scroll') .css('height', '66vh') + .css('background-color', '#fff') + .css('padding-top', '12px') .append(container)))) .append($('
    ') .css('display', 'flex') @@ -2274,7 +2280,10 @@ function MetadataButtons() { $(dialog).modal('hide'); }); }); - const toolbar = $('
    '); + const toolbar = $('
    ') + .css('display', 'flex') + .css('align-items', 'flex-end') + .css('margin-bottom', '10px'); const container = $('
      ').css('padding', '0 20px'); dialog .append($('
      ') @@ -2289,6 +2298,8 @@ function MetadataButtons() { .append($('
      ') .css('overflow-y', 'scroll') .css('height', '70vh') + .css('background-color', '#fff') + .css('padding-top', '12px') .append(container)))) .append($('
      ') .css('display', 'flex') diff --git a/addons/metadata/static/metadata-fields.js b/addons/metadata/static/metadata-fields.js index 3aa277629f4..0e4f4d8152e 100644 --- a/addons/metadata/static/metadata-fields.js +++ b/addons/metadata/static/metadata-fields.js @@ -14,6 +14,8 @@ const $ = require('jquery'); // Style definitions const AUTOFILLED_BG_COLOR = '#fffbf0'; +const INPUT_BG_COLOR = '#f2f6fa'; +const INPUT_BORDER_COLOR = '#cad9e6'; const $osf = require('js/osfHelpers'); const fangorn = require('js/fangorn'); const rdmGettext = require('js/rdmGettext'); @@ -29,8 +31,145 @@ const sizeofFormat = require("./util").sizeofFormat; const getLocalizedText = util.getLocalizedText; const normalizeText = util.normalizeText; +function appendTagBadges(container, tags) { + tags.forEach(function(tag) { + container.append($('') + .addClass('label label-default metadata-group-tag') + .css('margin-left', '6px') + .text(getLocalizedText(tag))); + }); +} + var filteredPages = []; +function resolveUI(ui, questionFields) { + if (!Array.isArray(ui)) { + return ui; + } + var fallback = null; + for (var i = 0; i < ui.length; i++) { + var entry = ui[i]; + if (!entry.condition) { + fallback = entry; + continue; + } + if (evaluateCond(entry.condition, questionFields)) return entry; + } + return fallback; +} + +function PageHeader(ui) { + this.ui = ui; + this.element = $('
      ') + .addClass('metadata-page-header') + .css('margin-bottom', '12px'); +} + +PageHeader.prototype.refresh = function(questionFields) { + var resolved = resolveUI(this.ui, questionFields); + if (resolved && resolved.header) { + this.element.html(getLocalizedText(resolved.header)); + this.element.find('ul').css('padding-left', '18px'); + this.element.show(); + } else { + this.element.hide(); + } +}; + +function GroupContainer(def, isChild) { + this.id = def.id; + this.parent = def.parent || null; + this.isChild = isChild; + this.children = []; + this.checkMark = isChild ? null : $('') + .addClass('fa fa-check') + .css({ color: '#5cb85c', 'margin-left': '8px' }) + .hide(); + this.element = $('
      ').addClass('metadata-group'); + this._heading = $('
      ') + .addClass('metadata-group-heading') + .css('margin-bottom', '6px') + .css('margin-top', '18px'); + this.element.append(this._heading); + this.content = $('
      '); + if (def.bar) { + this.content.css('border-left', '3px solid #ddd') + .css('padding-left', '12px'); + } + this.element.append(this.content); + this._renderHeading(def); +} + +GroupContainer.prototype._renderHeading = function(def) { + this._heading.empty(); + if (def.marker === 'circle') { + this._heading.append($('').text('◯').css('margin-right', '6px')); + } + var title = $('').text(getLocalizedText(def.title)); + if (this.isChild) { + title.css('font-size', '13px'); + } + this._heading.append(title); + if (def.tags) { + appendTagBadges(this._heading, def.tags); + } + if (def.info) { + var infoMark = $('') + .text('\u24D8') + .css({ cursor: 'pointer', 'margin-left': '6px', color: '#5bc0de' }); + infoMark.popover({ + content: getLocalizedText(def.info), + html: true, + trigger: 'focus', + placement: 'bottom', + container: 'body' + }).attr('tabindex', '0'); + this._heading.append(infoMark); + } + if (this.checkMark) this._heading.append(this.checkMark); + if (def.help) { + this._heading.append($('

      ') + .addClass('text-muted') + .css('margin', '4px 0 0') + .css('font-size', '12px') + .html(getLocalizedText(def.help))); + } +}; + +GroupContainer.prototype.refresh = function(questionFields) { + // Update conditional heading + if (this._sourceUI) { + var resolved = resolveUI(this._sourceUI, questionFields); + if (resolved && resolved.group) { + var groupRef = resolved.group; + var def = typeof groupRef === 'object' ? groupRef : null; + if (def) { + this._renderHeading(def); + } + } + } + // Update visibility + var hasVisibleChild = this.content.children().toArray().some(function(child) { + return $(child).css('display') !== 'none'; + }); + if (hasVisibleChild) { + this.element.show(); + } else { + this.element.hide(); + } + // Update check mark (top-level groups only) + if (this.checkMark) { + var hasValidChild = this.children.some(function(field) { + return field.hasValidValue(); + }); + if (hasValidChild) { + this.checkMark.show(); + } else { + this.checkMark.hide(); + } + } +}; + const logPrefix = '[metadata] '; const QuestionPage = oop.defclass({ @@ -68,7 +207,9 @@ const QuestionPage = oop.defclass({ if (!self.questionFilter(question)) { return; } - const value = (fileItemData[question.qid] || {}).value; + const value = self.options.multiple + ? self.options.commonValues[question.qid] + : (fileItemData[question.qid] || {}).value; const field = createQuestionField( question, value, @@ -86,9 +227,89 @@ const QuestionPage = oop.defclass({ self.fields.push(field); }); }); + self._buildContainer(); return self.fields; }, + _buildContainer: function() { + const self = this; + self.container = $('
      '); + + // Page-level header note (A-7) + self.headers = []; + (self.schema.pages || []).forEach(function(page) { + if (!page.ui) return; + var header = new PageHeader(page.ui); + self.container.append(header.element); + self.headers.push(header); + }); + + const groupDefs = {}; // groupId -> groupDef + self.groupContainers = {}; // groupId -> GroupContainer + + var variantQid = self.schema.ui && self.schema.ui.variant; + var variantContainer = self.options.variantContainer; + if (variantContainer) variantContainer.empty(); + + function ensureGroup(groupId) { + if (self.groupContainers[groupId]) return; + var def = groupDefs[groupId]; + if (def.parent && !self.groupContainers[def.parent]) { + ensureGroup(def.parent); + } + var group = new GroupContainer(def, !!def.parent); + self.groupContainers[groupId] = group; + if (def.parent) { + self.groupContainers[def.parent].content.append(group.element); + } else { + self.container.append(group.element); + } + } + + self.fields.forEach(function(field) { + if (variantContainer && field.question.qid === variantQid) { + var label = $('