Skip to content

fix: data process and processing schema creation#40

Open
arjunsridhar12345 wants to merge 3 commits into
devfrom
37-fix-data-process-creation
Open

fix: data process and processing schema creation#40
arjunsridhar12345 wants to merge 3 commits into
devfrom
37-fix-data-process-creation

Conversation

@arjunsridhar12345

@arjunsridhar12345 arjunsridhar12345 commented Jul 14, 2026

Copy link
Copy Markdown
Collaborator

Attempts to fix #37. Adds the pipeline name to the data process object so it can be merged later in the metadata manager. Adding the pipeline name to the data process requires the same name to be present in the list of pipelines as well. Thus, that has been added here as well.

Sample output is shown below trying to test this fix:

{
   "object_type": "Processing",
   "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/processing.py",
   "schema_version": "2.3.0",
   "data_processes": [
      {
         "object_type": "Data process",
         "process_type": "Other",
         "name": "Other",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "",
            "name": null,
            "version": "",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": {
               "input_source": "//allen/aind/stage/vr-foraging/data/860898/860898_2026-07-09T193353Z/behavior"
            },
            "core_dependency": null
         },
         "experimenters": [
            "AIND Scientific Computing"
         ],
         "pipeline_name": "transform_and_upload_v2",
         "start_date_time": "2026-07-10T01:15:29Z",
         "end_date_time": "2026-07-10T01:15:30Z",
         "output_path": null,
         "output_parameters": null,
         "notes": "Data was copied",
         "resources": null
      },
      {
         "object_type": "Data process",
         "process_type": "Compression",
         "name": "Compression",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "ghcr.io/allenneuraldynamics/aind-behavior-video-transformation",
            "name": null,
            "version": "0.2.1",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": {
               "input_source": "//allen/aind/stage/vr-foraging/data/860898/860898_2026-07-09T193353Z/behavior-videos",
               "output_directory": "/allen/aind/stage/svc_aind_airflow/prod/860898_2026-07-09_19-33-53_aa0568f31b/behavior-videos"
            },
            "core_dependency": null
         },
         "experimenters": [
            "AIND Scientific Computing"
         ],
         "pipeline_name": "transform_and_upload_v2",
         "start_date_time": "2026-07-10T01:15:29Z",
         "end_date_time": "2026-07-10T11:36:16Z",
         "output_path": null,
         "output_parameters": null,
         "notes": null,
         "resources": null
      },
      {
         "object_type": "Data process",
         "process_type": "Pipeline",
         "name": "Dynamic foraging NWB packaging",
         "stage": "Processing",
         "code": {
            "object_type": "Code",
            "url": "https://github.com/AllenNeuralDynamics/dynamic-foraging-processing",
            "name": null,
            "version": "0.1.0",
            "commit_hash": null,
            "container": null,
            "run_script": null,
            "language": null,
            "language_version": null,
            "input_data": null,
            "parameters": null,
            "core_dependency": null
         },
         "experimenters": [
            "Alex Piet",
            "Micah Woodard",
            "Bruno Cruz",
            "Arjun Sridhar"
         ],
         "pipeline_name": "dynamic-foraging-processing-pipeline",
         "start_date_time": "2026-07-14T10:28:04.143454-07:00",
         "end_date_time": "2026-07-14T10:28:04.143454-07:00",
         "output_path": null,
         "output_parameters": null,
         "notes": null,
         "resources": null
      }
   ],
   "pipelines": [
      {
         "object_type": "Code",
         "url": "https://github.com/AllenNeuralDynamics/dynamic-foraging-processing",
         "name": "dynamic-foraging-processing-pipeline",
         "version": "0.1.0",
         "commit_hash": null,
         "container": null,
         "run_script": null,
         "language": null,
         "language_version": null,
         "input_data": null,
         "parameters": null,
         "core_dependency": null
      },
      {
         "object_type": "Code",
         "url": "https://github.com/AllenNeuralDynamics/aind-airflow-dags",
         "name": "transform_and_upload_v2",
         "version": "0.39.1",
         "commit_hash": null,
         "container": null,
         "run_script": null,
         "language": "Python",
         "language_version": "3.9.18",
         "input_data": null,
         "parameters": {
            "job_type": "vr_foraging_v2"
         },
         "core_dependency": null
      }
   ],
   "notes": "Processes were reordered by start_date_time",
   "dependency_graph": {
      "Dynamic foraging NWB packaging": [],
      "Other": [],
      "Compression": []
   }
}

@arjunsridhar12345
arjunsridhar12345 marked this pull request as ready for review July 20, 2026 22:14
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants