From 0c295d5832bc7049f3422ee95c31efb59c4e4297 Mon Sep 17 00:00:00 2001 From: mrivar Date: Tue, 28 Apr 2026 11:39:43 +0200 Subject: [PATCH 01/18] refactor: parameters into more atomic files --- .gitignore | 1 + openhexa/sdk/pipelines/parameter/__init__.py | 65 ++++ openhexa/sdk/pipelines/parameter/decorator.py | 314 +++++++++++++++++ .../{parameter.py => parameter/types.py} | 323 +----------------- openhexa/sdk/pipelines/parameter/widgets.py | 24 ++ 5 files changed, 405 insertions(+), 322 deletions(-) create mode 100644 openhexa/sdk/pipelines/parameter/__init__.py create mode 100644 openhexa/sdk/pipelines/parameter/decorator.py rename openhexa/sdk/pipelines/{parameter.py => parameter/types.py} (55%) create mode 100644 openhexa/sdk/pipelines/parameter/widgets.py diff --git a/.gitignore b/.gitignore index a3e86f32..db03d7e3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ *.py[cod] *$py.class +.idea/ # C extensions *.so diff --git a/openhexa/sdk/pipelines/parameter/__init__.py b/openhexa/sdk/pipelines/parameter/__init__.py new file mode 100644 index 00000000..3bd1961e --- /dev/null +++ b/openhexa/sdk/pipelines/parameter/__init__.py @@ -0,0 +1,65 @@ +"""Pipeline parameters classes and functions. + +See https://github.com/BLSQ/openhexa/wiki/Writing-OpenHEXA-pipelines#pipeline-parameters for more information. +""" + +from openhexa.sdk.pipelines.exceptions import InvalidParameterError, ParameterValueError + +from .decorator import FunctionWithParameter, Parameter, parameter, validate_parameters +from .types import ( + Boolean, + ConnectionParameterType, + CustomConnectionType, + DatasetType, + DHIS2ConnectionType, + FileType, + Float, + GCSConnectionType, + IASOConnectionType, + Integer, + ParameterType, + PostgreSQLConnectionType, + S3ConnectionType, + Secret, + SecretType, + StringType, + TYPES_BY_PYTHON_TYPE, +) +from .widgets import DHIS2Widget, IASOWidget + +__all__ = [ + # Decorator and core classes + "parameter", + "Parameter", + "FunctionWithParameter", + "validate_parameters", + # Type base classes + "ParameterType", + "ConnectionParameterType", + # Primitive types + "StringType", + "Boolean", + "Integer", + "Float", + # Connection types + "PostgreSQLConnectionType", + "S3ConnectionType", + "GCSConnectionType", + "DHIS2ConnectionType", + "IASOConnectionType", + "CustomConnectionType", + # Resource types + "DatasetType", + "FileType", + # Secret + "Secret", + "SecretType", + # Registry + "TYPES_BY_PYTHON_TYPE", + # Widgets + "DHIS2Widget", + "IASOWidget", + # Exceptions (re-exported for backward compat) + "InvalidParameterError", + "ParameterValueError", +] diff --git a/openhexa/sdk/pipelines/parameter/decorator.py b/openhexa/sdk/pipelines/parameter/decorator.py new file mode 100644 index 00000000..24a4a8c8 --- /dev/null +++ b/openhexa/sdk/pipelines/parameter/decorator.py @@ -0,0 +1,314 @@ +"""Parameter class, decorator, and validation logic for pipeline parameters.""" + +import typing + +from openhexa.sdk.datasets import Dataset +from openhexa.sdk.files import File +from openhexa.sdk.pipelines.exceptions import InvalidParameterError, ParameterValueError +from openhexa.sdk.pipelines.utils import validate_pipeline_parameter_code +from openhexa.sdk.workspaces.connection import ( + CustomConnection, + DHIS2Connection, + GCSConnection, + IASOConnection, + PostgreSQLConnection, + S3Connection, +) + +from .types import Boolean, DHIS2ConnectionType, IASOConnectionType, Secret, TYPES_BY_PYTHON_TYPE +from .widgets import DHIS2Widget, IASOWidget + + +class Parameter: + """Pipeline parameter class. Contains validation logic specs generation logic.""" + + def __init__( + self, + code: str, + *, + type: type[ + str + | int + | bool + | float + | Secret + | DHIS2Connection + | IASOConnection + | PostgreSQLConnection + | GCSConnection + | S3Connection + | CustomConnection + | Dataset + | File + ], + name: str | None = None, + choices: typing.Sequence | None = None, + help: str | None = None, + default: typing.Any | None = None, + widget: DHIS2Widget | IASOWidget | None = None, + connection: str | None = None, + required: bool = True, + multiple: bool = False, + directory: str | None = None, + ): + validate_pipeline_parameter_code(code) + self.code = code + + try: + self.type = TYPES_BY_PYTHON_TYPE[type.__name__]() + except (KeyError, AttributeError): + valid_parameter_types = [k for k in TYPES_BY_PYTHON_TYPE.keys()] + raise InvalidParameterError( + f"Invalid parameter type provided ({type}). " + f"Valid parameter types are {', '.join(valid_parameter_types)}" + ) + + if choices is not None: + if not self.type.accepts_choices: + raise InvalidParameterError(f"Parameters of type {self.type} don't accept choices.") + elif len(choices) == 0: + raise InvalidParameterError("Choices, if provided, cannot be empty.") + + try: + for choice in choices: + self.type.validate(choice) + except ParameterValueError: + raise InvalidParameterError(f"The provided choices are not valid for the {self.type} parameter type.") + self.choices = choices + + self.name = name + self.help = help + self.required = required + + if multiple is True and not self.type.accepts_multiple: + raise InvalidParameterError(f"Parameters of type {self.type} can't have multiple values.") + self.multiple = multiple + + self.widget = widget + self.connection = connection + self.directory = directory + + self._validate_default(default, multiple) + self.default = default + + def validate(self, value: typing.Any) -> typing.Any: + """Validate the provided value against the parameter, taking required / default options into account.""" + if self.multiple: + return self._validate_multiple(value) + else: + return self._validate_single(value) + + def to_dict(self) -> dict[str, typing.Any]: + """Return a dictionary representation of the Parameter instance.""" + return { + "code": self.code, + "type": self.type.spec_type, + "name": self.name, + "choices": self.choices, + "help": self.help, + "default": self.default, + "widget": self.widget.value if self.widget else None, + "connection": self.connection, + "required": self.required, + "multiple": self.multiple, + "directory": self.directory, + } + + def _validate_single(self, value: typing.Any): + # Normalize empty values to None and handles default + normalized_value = self.type.normalize(value) + if normalized_value is None and self.default is not None: + normalized_value = self.default + + if normalized_value is None: + if isinstance(self.type, Boolean): + normalized_value = False + elif self.required: + raise ParameterValueError(f"{self.code} is required") + else: + return None + + pre_validated = self.type.validate(normalized_value) + if self.choices is not None and pre_validated not in self.choices: + raise ParameterValueError(f"The provided value for {self.code} is not included in the provided choices.") + + return pre_validated + + def _validate_multiple(self, value: typing.Any): + # Reject values that are not lists + if value is not None and not isinstance(value, list): + raise InvalidParameterError("If provided, value should be a list when parameter is multiple.") + + # Normalize empty values to an empty list + if value is None: + normalized_value = [] + else: + normalized_value = [self.type.normalize(v) for v in value] + normalized_value = list(filter(lambda v: v is not None, normalized_value)) + if len(normalized_value) == 0 and self.default is not None: + normalized_value = self.default + + if len(normalized_value) == 0 and self.required: + raise ParameterValueError(f"{self.code} is required") + + pre_validated = [self.type.validate(single_value) for single_value in normalized_value] + if self.choices is not None and any(v not in self.choices for v in pre_validated): + raise ParameterValueError( + f"One of the provided values for {self.code} is not included in the provided choices." + ) + + return pre_validated + + def _validate_default(self, default: typing.Any, multiple: bool): + if default is None: + return + + try: + if multiple: + if not isinstance(default, list): + raise InvalidParameterError("Default values should be lists when using multiple=True") + for default_value in default: + self.type.validate_default(default_value) + else: + self.type.validate_default(default) + except ParameterValueError: + raise InvalidParameterError(f"The default value for {self.code} is not valid.") + + if self.choices is not None: + if isinstance(default, list): + if not all(d in self.choices for d in default): + raise InvalidParameterError( + f"The default list of values for {self.code} is not included in the provided choices." + ) + elif default not in self.choices: + raise InvalidParameterError( + f"The default value for {self.code} is not included in the provided choices." + ) + + +def validate_parameters(parameters: list[Parameter]): + """Validate the provided connection parameters if they relate to existing connection parameter.""" + supported_connection_types = {DHIS2ConnectionType, IASOConnectionType} + connection_parameters = {p.code for p in parameters if type(p.type) in supported_connection_types} + + for parameter in parameters: + if parameter.connection and parameter.connection not in connection_parameters: + raise InvalidParameterError( + f"Connection field '{parameter.code}' references a non-existing connection parameter '{parameter.connection}'" + ) + if ( + parameter.widget + and (parameter.widget in DHIS2Widget or parameter.widget in IASOWidget) + and not parameter.connection + ): + raise InvalidParameterError( + f"Widgets require a connection parameter. Please provide a connection parameter for {parameter.code}. " + f"Example: @parameter('my_connection', ...)" + f"Example: @parameter('{parameter.code}', widget = ..., connection='my_connection')" + ) + + +def parameter( + code: str, + *, + type: type[ + str + | int + | bool + | float + | Secret + | DHIS2Connection + | IASOConnection + | PostgreSQLConnection + | GCSConnection + | S3Connection + | CustomConnection + | Dataset + | File + ], + name: str | None = None, + choices: typing.Sequence | None = None, + help: str | None = None, + widget: DHIS2Widget | IASOWidget | None = None, + connection: str | None = None, + default: typing.Any | None = None, + required: bool = True, + multiple: bool = False, + directory: str | None = None, +): + """Decorate a pipeline function by attaching a parameter to it.. + + This decorator must be used on a function decorated by the @pipeline decorator. + + Parameters + ---------- + code : str + The parameter identifier (must be unique for a given pipeline) + type : {str, int, bool, float, DHIS2Connection, IASOConnection, PostgreSQLConnection, GCSConnection, S3Connection, CustomConnection, Dataset, File} + The parameter Python type + name : str, optional + A name for the parameter (will be used instead of the code in the web interface) + choices : list, optional + An optional list or tuple of choices for the parameter (will be used to build a choice widget in the web + interface) + help : str, optional + An optional help text to be displayed in the web interface + widget : DHIS2Widget|IASOWidget, optional + An optional widget type for the parameter (only used if the parameter type is DHIS2Connection, IASOConnection) + connection : str, optional + An optional connection parameter that will be used to link widget to the connection. + default : any, optional + An optional default value for the parameter (should be of the type defined by the type parameter) + required : bool, default=True + Whether the parameter is mandatory + multiple : bool, default=True + Whether this parameter should be provided multiple values (if True, the value must be provided as a list of + values of the chosen type) + directory : str, optional + An optional parameter to force file selection to specific directory (only used for parameter type File). If the directory does not exist, it will be ignored. + + Returns + ------- + typing.Callable + A decorator that returns the Pipeline with the parameter attached + + """ + + def decorator(fun): + return FunctionWithParameter( + fun, + Parameter( + code, + type=type, + name=name, + choices=choices, + help=help, + default=default, + required=required, + widget=widget, + connection=connection, + multiple=multiple, + directory=directory, + ), + ) + + return decorator + + +class FunctionWithParameter: + """Wrapper class for pipeline functions decorated with the @parameter decorator.""" + + def __init__(self, function, added_parameter: Parameter): + self.function = function + self.parameter = added_parameter + + def get_all_parameters(self) -> list[Parameter]: + """Go through the decorators chain to find all pipeline parameters.""" + if isinstance(self.function, FunctionWithParameter): + return [self.parameter, *self.function.get_all_parameters()] + + return [self.parameter] + + def __call__(self, *args, **kwargs): + """Call the decorated pipeline function.""" + return self.function(*args, **kwargs) diff --git a/openhexa/sdk/pipelines/parameter.py b/openhexa/sdk/pipelines/parameter/types.py similarity index 55% rename from openhexa/sdk/pipelines/parameter.py rename to openhexa/sdk/pipelines/parameter/types.py index 4e6938ed..ff06e39c 100644 --- a/openhexa/sdk/pipelines/parameter.py +++ b/openhexa/sdk/pipelines/parameter/types.py @@ -1,15 +1,10 @@ -"""Pipeline parameters classes and functions. - -See https://github.com/BLSQ/openhexa/wiki/Writing-OpenHEXA-pipelines#pipeline-parameters for more information. -""" +"""Parameter type classes for pipeline parameters.""" import typing -from enum import StrEnum from openhexa.sdk.datasets import Dataset from openhexa.sdk.files import File from openhexa.sdk.pipelines.exceptions import InvalidParameterError, ParameterValueError -from openhexa.sdk.pipelines.utils import validate_pipeline_parameter_code from openhexa.sdk.workspaces import workspace from openhexa.sdk.workspaces.connection import ( Connection, @@ -469,319 +464,3 @@ def validate_default(self, value: typing.Any | None): "Dataset": DatasetType, "File": FileType, } - - -class IASOWidget(StrEnum): - """Enum for IASO widgets.""" - - IASO_FORMS = "IASO_FORMS" - IASO_ORG_UNITS = "IASO_ORG_UNITS" - IASO_PROJECTS = "IASO_PROJECTS" - - -class DHIS2Widget(StrEnum): - """Enum for DHIS2 widgets.""" - - ORG_UNITS = "DHIS2_ORG_UNITS" - ORG_UNIT_GROUPS = "DHIS2_ORG_UNIT_GROUPS" - ORG_UNIT_LEVELS = "DHIS2_ORG_UNIT_LEVELS" - DATASETS = "DHIS2_DATASETS" - DATA_ELEMENTS = "DHIS2_DATA_ELEMENTS" - DATA_ELEMENT_GROUPS = "DHIS2_DATA_ELEMENT_GROUPS" - INDICATORS = "DHIS2_INDICATORS" - INDICATOR_GROUPS = "DHIS2_INDICATOR_GROUPS" - - -class Parameter: - """Pipeline parameter class. Contains validation logic specs generation logic.""" - - def __init__( - self, - code: str, - *, - type: type[ - str - | int - | bool - | float - | Secret - | DHIS2Connection - | IASOConnection - | PostgreSQLConnection - | GCSConnection - | S3Connection - | CustomConnection - | Dataset - | File - ], - name: str | None = None, - choices: typing.Sequence | None = None, - help: str | None = None, - default: typing.Any | None = None, - widget: DHIS2Widget | IASOWidget | None = None, - connection: str | None = None, - required: bool = True, - multiple: bool = False, - directory: str | None = None, - ): - validate_pipeline_parameter_code(code) - self.code = code - - try: - self.type = TYPES_BY_PYTHON_TYPE[type.__name__]() - except (KeyError, AttributeError): - valid_parameter_types = [k for k in TYPES_BY_PYTHON_TYPE.keys()] - raise InvalidParameterError( - f"Invalid parameter type provided ({type}). " - f"Valid parameter types are {', '.join(valid_parameter_types)}" - ) - - if choices is not None: - if not self.type.accepts_choices: - raise InvalidParameterError(f"Parameters of type {self.type} don't accept choices.") - elif len(choices) == 0: - raise InvalidParameterError("Choices, if provided, cannot be empty.") - - try: - for choice in choices: - self.type.validate(choice) - except ParameterValueError: - raise InvalidParameterError(f"The provided choices are not valid for the {self.type} parameter type.") - self.choices = choices - - self.name = name - self.help = help - self.required = required - - if multiple is True and not self.type.accepts_multiple: - raise InvalidParameterError(f"Parameters of type {self.type} can't have multiple values.") - self.multiple = multiple - - self.widget = widget - self.connection = connection - self.directory = directory - - self._validate_default(default, multiple) - self.default = default - - def validate(self, value: typing.Any) -> typing.Any: - """Validate the provided value against the parameter, taking required / default options into account.""" - if self.multiple: - return self._validate_multiple(value) - else: - return self._validate_single(value) - - def to_dict(self) -> dict[str, typing.Any]: - """Return a dictionary representation of the Parameter instance.""" - return { - "code": self.code, - "type": self.type.spec_type, - "name": self.name, - "choices": self.choices, - "help": self.help, - "default": self.default, - "widget": self.widget.value if self.widget else None, - "connection": self.connection, - "required": self.required, - "multiple": self.multiple, - "directory": self.directory, - } - - def _validate_single(self, value: typing.Any): - # Normalize empty values to None and handles default - normalized_value = self.type.normalize(value) - if normalized_value is None and self.default is not None: - normalized_value = self.default - - if normalized_value is None: - if isinstance(self.type, Boolean): - normalized_value = False - elif self.required: - raise ParameterValueError(f"{self.code} is required") - else: - return None - - pre_validated = self.type.validate(normalized_value) - if self.choices is not None and pre_validated not in self.choices: - raise ParameterValueError(f"The provided value for {self.code} is not included in the provided choices.") - - return pre_validated - - def _validate_multiple(self, value: typing.Any): - # Reject values that are not lists - if value is not None and not isinstance(value, list): - raise InvalidParameterError("If provided, value should be a list when parameter is multiple.") - - # Normalize empty values to an empty list - if value is None: - normalized_value = [] - else: - normalized_value = [self.type.normalize(v) for v in value] - normalized_value = list(filter(lambda v: v is not None, normalized_value)) - if len(normalized_value) == 0 and self.default is not None: - normalized_value = self.default - - if len(normalized_value) == 0 and self.required: - raise ParameterValueError(f"{self.code} is required") - - pre_validated = [self.type.validate(single_value) for single_value in normalized_value] - if self.choices is not None and any(v not in self.choices for v in pre_validated): - raise ParameterValueError( - f"One of the provided values for {self.code} is not included in the provided choices." - ) - - return pre_validated - - def _validate_default(self, default: typing.Any, multiple: bool): - if default is None: - return - - try: - if multiple: - if not isinstance(default, list): - raise InvalidParameterError("Default values should be lists when using multiple=True") - for default_value in default: - self.type.validate_default(default_value) - else: - self.type.validate_default(default) - except ParameterValueError: - raise InvalidParameterError(f"The default value for {self.code} is not valid.") - - if self.choices is not None: - if isinstance(default, list): - if not all(d in self.choices for d in default): - raise InvalidParameterError( - f"The default list of values for {self.code} is not included in the provided choices." - ) - elif default not in self.choices: - raise InvalidParameterError( - f"The default value for {self.code} is not included in the provided choices." - ) - - -def validate_parameters(parameters: list[Parameter]): - """Validate the provided connection parameters if they relate to existing connection parameter.""" - supported_connection_types = {DHIS2ConnectionType, IASOConnectionType} - connection_parameters = {p.code for p in parameters if type(p.type) in supported_connection_types} - - for parameter in parameters: - if parameter.connection and parameter.connection not in connection_parameters: - raise InvalidParameterError( - f"Connection field '{parameter.code}' references a non-existing connection parameter '{parameter.connection}'" - ) - if ( - parameter.widget - and (parameter.widget in DHIS2Widget or parameter.widget in IASOWidget) - and not parameter.connection - ): - raise InvalidParameterError( - f"Widgets require a connection parameter. Please provide a connection parameter for {parameter.code}. " - f"Example: @parameter('my_connection', ...)" - f"Example: @parameter('{parameter.code}', widget = ..., connection='my_connection')" - ) - - -def parameter( - code: str, - *, - type: type[ - str - | int - | bool - | float - | Secret - | DHIS2Connection - | IASOConnection - | PostgreSQLConnection - | GCSConnection - | S3Connection - | CustomConnection - | Dataset - | File - ], - name: str | None = None, - choices: typing.Sequence | None = None, - help: str | None = None, - widget: DHIS2Widget | IASOWidget | None = None, - connection: str | None = None, - default: typing.Any | None = None, - required: bool = True, - multiple: bool = False, - directory: str | None = None, -): - """Decorate a pipeline function by attaching a parameter to it.. - - This decorator must be used on a function decorated by the @pipeline decorator. - - Parameters - ---------- - code : str - The parameter identifier (must be unique for a given pipeline) - type : {str, int, bool, float, DHIS2Connection, IASOConnection, PostgreSQLConnection, GCSConnection, S3Connection, CustomConnection, Dataset, File} - The parameter Python type - name : str, optional - A name for the parameter (will be used instead of the code in the web interface) - choices : list, optional - An optional list or tuple of choices for the parameter (will be used to build a choice widget in the web - interface) - help : str, optional - An optional help text to be displayed in the web interface - widget : DHIS2Widget|IASOWidget, optional - An optional widget type for the parameter (only used if the parameter type is DHIS2Connection, IASOConnection) - connection : str, optional - An optional connection parameter that will be used to link widget to the connection. - default : any, optional - An optional default value for the parameter (should be of the type defined by the type parameter) - required : bool, default=True - Whether the parameter is mandatory - multiple : bool, default=True - Whether this parameter should be provided multiple values (if True, the value must be provided as a list of - values of the chosen type) - directory : str, optional - An optional parameter to force file selection to specific directory (only used for parameter type File). If the directory does not exist, it will be ignored. - - Returns - ------- - typing.Callable - A decorator that returns the Pipeline with the parameter attached - - """ - - def decorator(fun): - return FunctionWithParameter( - fun, - Parameter( - code, - type=type, - name=name, - choices=choices, - help=help, - default=default, - required=required, - widget=widget, - connection=connection, - multiple=multiple, - directory=directory, - ), - ) - - return decorator - - -class FunctionWithParameter: - """Wrapper class for pipeline functions decorated with the @parameter decorator.""" - - def __init__(self, function, added_parameter: Parameter): - self.function = function - self.parameter = added_parameter - - def get_all_parameters(self) -> list[Parameter]: - """Go through the decorators chain to find all pipeline parameters.""" - if isinstance(self.function, FunctionWithParameter): - return [self.parameter, *self.function.get_all_parameters()] - - return [self.parameter] - - def __call__(self, *args, **kwargs): - """Call the decorated pipeline function.""" - return self.function(*args, **kwargs) diff --git a/openhexa/sdk/pipelines/parameter/widgets.py b/openhexa/sdk/pipelines/parameter/widgets.py new file mode 100644 index 00000000..8ff6f1bf --- /dev/null +++ b/openhexa/sdk/pipelines/parameter/widgets.py @@ -0,0 +1,24 @@ +"""Widget enums for DHIS2 and IASO pipeline parameters.""" + +from enum import StrEnum + + +class IASOWidget(StrEnum): + """Enum for IASO widgets.""" + + IASO_FORMS = "IASO_FORMS" + IASO_ORG_UNITS = "IASO_ORG_UNITS" + IASO_PROJECTS = "IASO_PROJECTS" + + +class DHIS2Widget(StrEnum): + """Enum for DHIS2 widgets.""" + + ORG_UNITS = "DHIS2_ORG_UNITS" + ORG_UNIT_GROUPS = "DHIS2_ORG_UNIT_GROUPS" + ORG_UNIT_LEVELS = "DHIS2_ORG_UNIT_LEVELS" + DATASETS = "DHIS2_DATASETS" + DATA_ELEMENTS = "DHIS2_DATA_ELEMENTS" + DATA_ELEMENT_GROUPS = "DHIS2_DATA_ELEMENT_GROUPS" + INDICATORS = "DHIS2_INDICATORS" + INDICATOR_GROUPS = "DHIS2_INDICATOR_GROUPS" From 3d5b0110f01b3d2ada2bf75976b9f063028a63fe Mon Sep 17 00:00:00 2001 From: mrivar Date: Fri, 1 May 2026 14:32:14 +0200 Subject: [PATCH 02/18] feature: dynamic choices using FileChoices - first iteration --- openhexa/sdk/pipelines/parameter/__init__.py | 3 + openhexa/sdk/pipelines/parameter/choices.py | 52 +++++ openhexa/sdk/pipelines/parameter/decorator.py | 43 ++-- openhexa/sdk/pipelines/runtime.py | 18 +- tests/test_choices.py | 191 ++++++++++++++++++ 5 files changed, 291 insertions(+), 16 deletions(-) create mode 100644 openhexa/sdk/pipelines/parameter/choices.py create mode 100644 tests/test_choices.py diff --git a/openhexa/sdk/pipelines/parameter/__init__.py b/openhexa/sdk/pipelines/parameter/__init__.py index 3bd1961e..2511aa5a 100644 --- a/openhexa/sdk/pipelines/parameter/__init__.py +++ b/openhexa/sdk/pipelines/parameter/__init__.py @@ -5,6 +5,7 @@ from openhexa.sdk.pipelines.exceptions import InvalidParameterError, ParameterValueError +from .choices import FileChoices from .decorator import FunctionWithParameter, Parameter, parameter, validate_parameters from .types import ( Boolean, @@ -56,6 +57,8 @@ "SecretType", # Registry "TYPES_BY_PYTHON_TYPE", + # Dynamic choices + "FileChoices", # Widgets "DHIS2Widget", "IASOWidget", diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py new file mode 100644 index 00000000..c211e54e --- /dev/null +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -0,0 +1,52 @@ +"""Dynamic choices classes for pipeline parameters.""" + +from openhexa.sdk.pipelines.exceptions import InvalidParameterError + +_SUPPORTED_FORMATS = {"csv", "json", "yaml", "yml"} + + +class FileChoices: + """Descriptor for choices loaded dynamically from a file in the workspace file system. + + The file format is inferred from the path extension (.csv, .json, .yaml, .yml). + For CSV files with a single column, that column is used automatically. + For CSV/JSON/YAML files with multiple columns/keys, `column` must be specified. + + Parameters + ---------- + path : str + Path to the file in the workspace file system (e.g. "data/districts.csv"). + column : str, optional + Column name (CSV) or key (JSON/YAML) to use as choice values. + Required when the file has more than one column/key. + """ + + def __init__(self, path: str, column: str | None = None): + self.path = path + self.column = column + self.format = self._detect_format(path) + self.validate_spec() + + def _detect_format(self, path: str) -> str: + if not path or not isinstance(path, str): + raise InvalidParameterError("FileChoices path must be a non-empty string.") + ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" + if ext not in _SUPPORTED_FORMATS: + raise InvalidParameterError( + f"Cannot determine file format from path '{path}'. " + f"Supported extensions: {', '.join(sorted(_SUPPORTED_FORMATS))}." + ) + return "yaml" if ext == "yml" else ext + + def validate_spec(self): + if not self.path or not isinstance(self.path, str): + raise InvalidParameterError("FileChoices path must be a non-empty string.") + if self.column is not None and not isinstance(self.column, str): + raise InvalidParameterError("FileChoices column must be a string.") + + def to_dict(self) -> dict: + return { + "format": self.format, + "path": self.path, + "column": self.column, + } diff --git a/openhexa/sdk/pipelines/parameter/decorator.py b/openhexa/sdk/pipelines/parameter/decorator.py index 24a4a8c8..e99058f7 100644 --- a/openhexa/sdk/pipelines/parameter/decorator.py +++ b/openhexa/sdk/pipelines/parameter/decorator.py @@ -15,6 +15,7 @@ S3Connection, ) +from .choices import FileChoices from .types import Boolean, DHIS2ConnectionType, IASOConnectionType, Secret, TYPES_BY_PYTHON_TYPE from .widgets import DHIS2Widget, IASOWidget @@ -42,7 +43,7 @@ def __init__( | File ], name: str | None = None, - choices: typing.Sequence | None = None, + choices: typing.Sequence | FileChoices | None = None, help: str | None = None, default: typing.Any | None = None, widget: DHIS2Widget | IASOWidget | None = None, @@ -66,14 +67,19 @@ def __init__( if choices is not None: if not self.type.accepts_choices: raise InvalidParameterError(f"Parameters of type {self.type} don't accept choices.") - elif len(choices) == 0: - raise InvalidParameterError("Choices, if provided, cannot be empty.") - - try: - for choice in choices: - self.type.validate(choice) - except ParameterValueError: - raise InvalidParameterError(f"The provided choices are not valid for the {self.type} parameter type.") + if isinstance(choices, FileChoices): + # validate_spec() already ran in FileChoices.__init__; nothing more to check here + pass + else: + if len(choices) == 0: + raise InvalidParameterError("Choices, if provided, cannot be empty.") + try: + for choice in choices: + self.type.validate(choice) + except ParameterValueError: + raise InvalidParameterError( + f"The provided choices are not valid for the {self.type} parameter type." + ) self.choices = choices self.name = name @@ -100,11 +106,11 @@ def validate(self, value: typing.Any) -> typing.Any: def to_dict(self) -> dict[str, typing.Any]: """Return a dictionary representation of the Parameter instance.""" - return { + d = { "code": self.code, "type": self.type.spec_type, "name": self.name, - "choices": self.choices, + "choices": None if isinstance(self.choices, FileChoices) else self.choices, "help": self.help, "default": self.default, "widget": self.widget.value if self.widget else None, @@ -113,6 +119,9 @@ def to_dict(self) -> dict[str, typing.Any]: "multiple": self.multiple, "directory": self.directory, } + if isinstance(self.choices, FileChoices): + d["file_choices"] = self.choices.to_dict() + return d def _validate_single(self, value: typing.Any): # Normalize empty values to None and handles default @@ -129,7 +138,7 @@ def _validate_single(self, value: typing.Any): return None pre_validated = self.type.validate(normalized_value) - if self.choices is not None and pre_validated not in self.choices: + if self.choices is not None and not isinstance(self.choices, FileChoices) and pre_validated not in self.choices: raise ParameterValueError(f"The provided value for {self.code} is not included in the provided choices.") return pre_validated @@ -152,7 +161,11 @@ def _validate_multiple(self, value: typing.Any): raise ParameterValueError(f"{self.code} is required") pre_validated = [self.type.validate(single_value) for single_value in normalized_value] - if self.choices is not None and any(v not in self.choices for v in pre_validated): + if ( + self.choices is not None + and not isinstance(self.choices, FileChoices) + and any(v not in self.choices for v in pre_validated) + ): raise ParameterValueError( f"One of the provided values for {self.code} is not included in the provided choices." ) @@ -174,7 +187,7 @@ def _validate_default(self, default: typing.Any, multiple: bool): except ParameterValueError: raise InvalidParameterError(f"The default value for {self.code} is not valid.") - if self.choices is not None: + if self.choices is not None and not isinstance(self.choices, FileChoices): if isinstance(default, list): if not all(d in self.choices for d in default): raise InvalidParameterError( @@ -227,7 +240,7 @@ def parameter( | File ], name: str | None = None, - choices: typing.Sequence | None = None, + choices: typing.Sequence | FileChoices | None = None, help: str | None = None, widget: DHIS2Widget | IASOWidget | None = None, connection: str | None = None, diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index c1e3195e..434544ed 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -17,6 +17,7 @@ from openhexa.sdk.pipelines.parameter import ( TYPES_BY_PYTHON_TYPE, DHIS2Widget, + FileChoices, IASOWidget, Parameter, validate_parameters, @@ -172,6 +173,21 @@ def _get_decorator_arg_value(decorator: ast.Call, arg: Argument, index: int) -> return (keyword.value.id, True) elif isinstance(keyword.value, ast.List): return ([el.value for el in keyword.value.elts], True) + elif isinstance(keyword.value, ast.Call): + func = keyword.value.func + func_name = func.id if isinstance(func, ast.Name) else None + if func_name != "FileChoices": + raise ValueError(f"Unsupported call in choices argument: {func_name}") + # Extract positional arg (path) and keyword args (column, format override) + pos_args = [a.value for a in keyword.value.args if isinstance(a, ast.Constant)] + kw_args = { + kw.arg: kw.value.value + for kw in keyword.value.keywords + if isinstance(kw.value, ast.Constant) + } + if pos_args: + kw_args.setdefault("path", pos_args[0]) + return (FileChoices(**kw_args), True) elif isinstance(keyword.value, ast.Attribute): if keyword.value.attr in DHIS2Widget.__members__: return getattr(DHIS2Widget, keyword.value.attr), True @@ -287,7 +303,7 @@ def get_pipeline(pipeline_path: Path) -> Pipeline: Argument("code", [ast.Constant]), Argument("type", [ast.Name]), Argument("name", [ast.Constant]), - Argument("choices", [ast.List]), + Argument("choices", [ast.List, ast.Call]), Argument("help", [ast.Constant]), Argument("default", [ast.Constant, ast.List]), Argument("widget", [ast.Attribute]), diff --git a/tests/test_choices.py b/tests/test_choices.py new file mode 100644 index 00000000..5294ad80 --- /dev/null +++ b/tests/test_choices.py @@ -0,0 +1,191 @@ +"""Tests for FileChoices dynamic parameter choices.""" + +import tempfile +from unittest import TestCase + +import pytest + +from openhexa.sdk.pipelines.exceptions import InvalidParameterError, ParameterValueError +from openhexa.sdk.pipelines.parameter import FileChoices, Parameter, parameter +from openhexa.sdk.pipelines.parameter.choices import _SUPPORTED_FORMATS +from openhexa.sdk.pipelines.runtime import get_pipeline + + +# --------------------------------------------------------------------------- +# FileChoices construction +# --------------------------------------------------------------------------- + + +class TestFileChoicesConstruction: + def test_csv_auto_detected(self): + fc = FileChoices("districts.csv") + assert fc.format == "csv" + assert fc.path == "districts.csv" + assert fc.column is None + + def test_json_auto_detected(self): + fc = FileChoices("data/regions.json", column="code") + assert fc.format == "json" + assert fc.column == "code" + + def test_yaml_auto_detected(self): + assert FileChoices("list.yaml").format == "yaml" + + def test_yml_normalised_to_yaml(self): + assert FileChoices("list.yml").format == "yaml" + + def test_unsupported_extension_raises(self): + with pytest.raises(InvalidParameterError, match="Supported extensions"): + FileChoices("districts.xlsx") + + def test_no_extension_raises(self): + with pytest.raises(InvalidParameterError, match="Supported extensions"): + FileChoices("districts") + + def test_empty_path_raises(self): + with pytest.raises(InvalidParameterError): + FileChoices("") + + def test_non_string_column_raises(self): + with pytest.raises(InvalidParameterError): + FileChoices("districts.csv", column=42) + + def test_to_dict(self): + fc = FileChoices("data/districts.csv", column="code") + assert fc.to_dict() == {"format": "csv", "path": "data/districts.csv", "column": "code"} + + def test_to_dict_no_column(self): + fc = FileChoices("districts.csv") + assert fc.to_dict() == {"format": "csv", "path": "districts.csv", "column": None} + + +# --------------------------------------------------------------------------- +# Parameter integration +# --------------------------------------------------------------------------- + + +class TestParameterWithFileChoices: + def test_accepts_file_choices(self): + p = Parameter(code="district", type=str, choices=FileChoices("districts.csv")) + assert isinstance(p.choices, FileChoices) + + def test_to_dict_emits_file_choices_key(self): + p = Parameter(code="district", type=str, choices=FileChoices("districts.csv", column="code")) + d = p.to_dict() + assert d["choices"] is None + assert d["file_choices"] == {"format": "csv", "path": "districts.csv", "column": "code"} + + def test_to_dict_no_file_choices_key_for_static_choices(self): + p = Parameter(code="country", type=str, choices=["UG", "KE"]) + d = p.to_dict() + assert d["choices"] == ["UG", "KE"] + assert "file_choices" not in d + + def test_rejects_file_choices_on_bool_type(self): + with pytest.raises(InvalidParameterError, match="don't accept choices"): + Parameter(code="flag", type=bool, choices=FileChoices("flags.csv")) + + def test_validate_single_skips_choices_check(self): + p = Parameter(code="district", type=str, choices=FileChoices("districts.csv")) + # Any string value passes — the platform validates against the resolved list + assert p.validate("any_value") == "any_value" + + def test_validate_multiple_skips_choices_check(self): + p = Parameter(code="district", type=str, choices=FileChoices("districts.csv"), multiple=True) + assert p.validate(["A", "B", "C"]) == ["A", "B", "C"] + + def test_default_not_validated_against_file_choices(self): + # Should not raise even though default isn't in any resolved list + p = Parameter(code="district", type=str, choices=FileChoices("districts.csv"), default="UNKNOWN") + assert p.default == "UNKNOWN" + + def test_decorator_with_file_choices(self): + @parameter(code="district", type=str, choices=FileChoices("districts.csv")) + def my_pipeline(district): + pass + + params = my_pipeline.get_all_parameters() + assert len(params) == 1 + assert isinstance(params[0].choices, FileChoices) + + +# --------------------------------------------------------------------------- +# AST round-trip +# --------------------------------------------------------------------------- + + +class TestAstFileChoices(TestCase): + def _write_pipeline(self, tmpdir, param_line): + with open(f"{tmpdir}/pipeline.py", "w") as f: + f.write( + "\n".join( + [ + "from openhexa.sdk.pipelines import pipeline, parameter", + "from openhexa.sdk.pipelines.parameter import FileChoices", + "", + param_line, + "@pipeline(name='Test pipeline')", + "def test_pipeline(district):", + " pass", + ] + ) + ) + + def test_file_choices_csv_positional_path(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices=FileChoices('districts.csv'))", + ) + p = get_pipeline(tmpdir) + param_dict = p.to_dict()["parameters"][0] + assert param_dict["choices"] is None + assert param_dict["file_choices"] == {"format": "csv", "path": "districts.csv", "column": None} + + def test_file_choices_csv_with_column(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices=FileChoices('data/districts.csv', column='code'))", + ) + p = get_pipeline(tmpdir) + param_dict = p.to_dict()["parameters"][0] + assert param_dict["file_choices"] == {"format": "csv", "path": "data/districts.csv", "column": "code"} + + def test_file_choices_json(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices=FileChoices('regions.json', column='id'))", + ) + p = get_pipeline(tmpdir) + param_dict = p.to_dict()["parameters"][0] + assert param_dict["file_choices"]["format"] == "json" + + def test_file_choices_yaml(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices=FileChoices('list.yml'))", + ) + p = get_pipeline(tmpdir) + param_dict = p.to_dict()["parameters"][0] + assert param_dict["file_choices"]["format"] == "yaml" + + def test_unsupported_call_in_choices_raises(self): + with tempfile.TemporaryDirectory() as tmpdir: + with open(f"{tmpdir}/pipeline.py", "w") as f: + f.write( + "\n".join( + [ + "from openhexa.sdk.pipelines import pipeline, parameter", + "", + "@parameter('district', type=str, choices=dict(a=1))", + "@pipeline(name='Test pipeline')", + "def test_pipeline(district):", + " pass", + ] + ) + ) + with self.assertRaises(ValueError, msg="Unsupported call"): + get_pipeline(tmpdir) From f796235aae0229fb9f9d621ddc03d7a53e674862 Mon Sep 17 00:00:00 2001 From: mrivar Date: Fri, 1 May 2026 14:36:37 +0200 Subject: [PATCH 03/18] fix: lint --- openhexa/sdk/pipelines/parameter/__init__.py | 2 +- openhexa/sdk/pipelines/parameter/decorator.py | 2 +- openhexa/sdk/pipelines/runtime.py | 4 +--- tests/test_choices.py | 4 +--- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/__init__.py b/openhexa/sdk/pipelines/parameter/__init__.py index 2511aa5a..513b35d0 100644 --- a/openhexa/sdk/pipelines/parameter/__init__.py +++ b/openhexa/sdk/pipelines/parameter/__init__.py @@ -8,6 +8,7 @@ from .choices import FileChoices from .decorator import FunctionWithParameter, Parameter, parameter, validate_parameters from .types import ( + TYPES_BY_PYTHON_TYPE, Boolean, ConnectionParameterType, CustomConnectionType, @@ -24,7 +25,6 @@ Secret, SecretType, StringType, - TYPES_BY_PYTHON_TYPE, ) from .widgets import DHIS2Widget, IASOWidget diff --git a/openhexa/sdk/pipelines/parameter/decorator.py b/openhexa/sdk/pipelines/parameter/decorator.py index e99058f7..710f4c3d 100644 --- a/openhexa/sdk/pipelines/parameter/decorator.py +++ b/openhexa/sdk/pipelines/parameter/decorator.py @@ -16,7 +16,7 @@ ) from .choices import FileChoices -from .types import Boolean, DHIS2ConnectionType, IASOConnectionType, Secret, TYPES_BY_PYTHON_TYPE +from .types import TYPES_BY_PYTHON_TYPE, Boolean, DHIS2ConnectionType, IASOConnectionType, Secret from .widgets import DHIS2Widget, IASOWidget diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index 434544ed..b6876d63 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -181,9 +181,7 @@ def _get_decorator_arg_value(decorator: ast.Call, arg: Argument, index: int) -> # Extract positional arg (path) and keyword args (column, format override) pos_args = [a.value for a in keyword.value.args if isinstance(a, ast.Constant)] kw_args = { - kw.arg: kw.value.value - for kw in keyword.value.keywords - if isinstance(kw.value, ast.Constant) + kw.arg: kw.value.value for kw in keyword.value.keywords if isinstance(kw.value, ast.Constant) } if pos_args: kw_args.setdefault("path", pos_args[0]) diff --git a/tests/test_choices.py b/tests/test_choices.py index 5294ad80..a2aa6b3a 100644 --- a/tests/test_choices.py +++ b/tests/test_choices.py @@ -5,12 +5,10 @@ import pytest -from openhexa.sdk.pipelines.exceptions import InvalidParameterError, ParameterValueError +from openhexa.sdk.pipelines.exceptions import InvalidParameterError from openhexa.sdk.pipelines.parameter import FileChoices, Parameter, parameter -from openhexa.sdk.pipelines.parameter.choices import _SUPPORTED_FORMATS from openhexa.sdk.pipelines.runtime import get_pipeline - # --------------------------------------------------------------------------- # FileChoices construction # --------------------------------------------------------------------------- From 1e22facba1a16c9d26c21dd596b0a4778a09422e Mon Sep 17 00:00:00 2001 From: mrivar Date: Mon, 4 May 2026 14:41:07 +0200 Subject: [PATCH 04/18] refactor: rename ChoicesFromFile --- openhexa/sdk/pipelines/parameter/__init__.py | 4 +- openhexa/sdk/pipelines/parameter/choices.py | 11 +-- openhexa/sdk/pipelines/parameter/decorator.py | 22 +++--- openhexa/sdk/pipelines/runtime.py | 6 +- tests/test_choices.py | 72 +++++++++---------- 5 files changed, 58 insertions(+), 57 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/__init__.py b/openhexa/sdk/pipelines/parameter/__init__.py index 513b35d0..774fd1ec 100644 --- a/openhexa/sdk/pipelines/parameter/__init__.py +++ b/openhexa/sdk/pipelines/parameter/__init__.py @@ -5,7 +5,7 @@ from openhexa.sdk.pipelines.exceptions import InvalidParameterError, ParameterValueError -from .choices import FileChoices +from .choices import ChoicesFromFile from .decorator import FunctionWithParameter, Parameter, parameter, validate_parameters from .types import ( TYPES_BY_PYTHON_TYPE, @@ -58,7 +58,7 @@ # Registry "TYPES_BY_PYTHON_TYPE", # Dynamic choices - "FileChoices", + "ChoicesFromFile", # Widgets "DHIS2Widget", "IASOWidget", diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py index c211e54e..56736fac 100644 --- a/openhexa/sdk/pipelines/parameter/choices.py +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -5,7 +5,7 @@ _SUPPORTED_FORMATS = {"csv", "json", "yaml", "yml"} -class FileChoices: +class ChoicesFromFile: """Descriptor for choices loaded dynamically from a file in the workspace file system. The file format is inferred from the path extension (.csv, .json, .yaml, .yml). @@ -27,9 +27,10 @@ def __init__(self, path: str, column: str | None = None): self.format = self._detect_format(path) self.validate_spec() - def _detect_format(self, path: str) -> str: + @staticmethod + def _detect_format(path: str) -> str: if not path or not isinstance(path, str): - raise InvalidParameterError("FileChoices path must be a non-empty string.") + raise InvalidParameterError("ChoicesFromFile path must be a non-empty string.") ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" if ext not in _SUPPORTED_FORMATS: raise InvalidParameterError( @@ -40,9 +41,9 @@ def _detect_format(self, path: str) -> str: def validate_spec(self): if not self.path or not isinstance(self.path, str): - raise InvalidParameterError("FileChoices path must be a non-empty string.") + raise InvalidParameterError("ChoicesFromFile path must be a non-empty string.") if self.column is not None and not isinstance(self.column, str): - raise InvalidParameterError("FileChoices column must be a string.") + raise InvalidParameterError("ChoicesFromFile column must be a string.") def to_dict(self) -> dict: return { diff --git a/openhexa/sdk/pipelines/parameter/decorator.py b/openhexa/sdk/pipelines/parameter/decorator.py index 710f4c3d..e90ed9b9 100644 --- a/openhexa/sdk/pipelines/parameter/decorator.py +++ b/openhexa/sdk/pipelines/parameter/decorator.py @@ -15,7 +15,7 @@ S3Connection, ) -from .choices import FileChoices +from .choices import ChoicesFromFile from .types import TYPES_BY_PYTHON_TYPE, Boolean, DHIS2ConnectionType, IASOConnectionType, Secret from .widgets import DHIS2Widget, IASOWidget @@ -43,7 +43,7 @@ def __init__( | File ], name: str | None = None, - choices: typing.Sequence | FileChoices | None = None, + choices: typing.Sequence | ChoicesFromFile | None = None, help: str | None = None, default: typing.Any | None = None, widget: DHIS2Widget | IASOWidget | None = None, @@ -67,8 +67,8 @@ def __init__( if choices is not None: if not self.type.accepts_choices: raise InvalidParameterError(f"Parameters of type {self.type} don't accept choices.") - if isinstance(choices, FileChoices): - # validate_spec() already ran in FileChoices.__init__; nothing more to check here + if isinstance(choices, ChoicesFromFile): + # validate_spec() already ran in ChoicesFromFile.__init__; nothing more to check here pass else: if len(choices) == 0: @@ -110,7 +110,7 @@ def to_dict(self) -> dict[str, typing.Any]: "code": self.code, "type": self.type.spec_type, "name": self.name, - "choices": None if isinstance(self.choices, FileChoices) else self.choices, + "choices": None if isinstance(self.choices, ChoicesFromFile) else self.choices, "help": self.help, "default": self.default, "widget": self.widget.value if self.widget else None, @@ -119,8 +119,8 @@ def to_dict(self) -> dict[str, typing.Any]: "multiple": self.multiple, "directory": self.directory, } - if isinstance(self.choices, FileChoices): - d["file_choices"] = self.choices.to_dict() + if isinstance(self.choices, ChoicesFromFile): + d["choices_from_file"] = self.choices.to_dict() return d def _validate_single(self, value: typing.Any): @@ -138,7 +138,7 @@ def _validate_single(self, value: typing.Any): return None pre_validated = self.type.validate(normalized_value) - if self.choices is not None and not isinstance(self.choices, FileChoices) and pre_validated not in self.choices: + if self.choices is not None and not isinstance(self.choices, ChoicesFromFile) and pre_validated not in self.choices: raise ParameterValueError(f"The provided value for {self.code} is not included in the provided choices.") return pre_validated @@ -163,7 +163,7 @@ def _validate_multiple(self, value: typing.Any): pre_validated = [self.type.validate(single_value) for single_value in normalized_value] if ( self.choices is not None - and not isinstance(self.choices, FileChoices) + and not isinstance(self.choices, ChoicesFromFile) and any(v not in self.choices for v in pre_validated) ): raise ParameterValueError( @@ -187,7 +187,7 @@ def _validate_default(self, default: typing.Any, multiple: bool): except ParameterValueError: raise InvalidParameterError(f"The default value for {self.code} is not valid.") - if self.choices is not None and not isinstance(self.choices, FileChoices): + if self.choices is not None and not isinstance(self.choices, ChoicesFromFile): if isinstance(default, list): if not all(d in self.choices for d in default): raise InvalidParameterError( @@ -240,7 +240,7 @@ def parameter( | File ], name: str | None = None, - choices: typing.Sequence | FileChoices | None = None, + choices: typing.Sequence | ChoicesFromFile | None = None, help: str | None = None, widget: DHIS2Widget | IASOWidget | None = None, connection: str | None = None, diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index b6876d63..399390e1 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -17,7 +17,7 @@ from openhexa.sdk.pipelines.parameter import ( TYPES_BY_PYTHON_TYPE, DHIS2Widget, - FileChoices, + ChoicesFromFile, IASOWidget, Parameter, validate_parameters, @@ -176,7 +176,7 @@ def _get_decorator_arg_value(decorator: ast.Call, arg: Argument, index: int) -> elif isinstance(keyword.value, ast.Call): func = keyword.value.func func_name = func.id if isinstance(func, ast.Name) else None - if func_name != "FileChoices": + if func_name != "ChoicesFromFile": raise ValueError(f"Unsupported call in choices argument: {func_name}") # Extract positional arg (path) and keyword args (column, format override) pos_args = [a.value for a in keyword.value.args if isinstance(a, ast.Constant)] @@ -185,7 +185,7 @@ def _get_decorator_arg_value(decorator: ast.Call, arg: Argument, index: int) -> } if pos_args: kw_args.setdefault("path", pos_args[0]) - return (FileChoices(**kw_args), True) + return (ChoicesFromFile(**kw_args), True) elif isinstance(keyword.value, ast.Attribute): if keyword.value.attr in DHIS2Widget.__members__: return getattr(DHIS2Widget, keyword.value.attr), True diff --git a/tests/test_choices.py b/tests/test_choices.py index a2aa6b3a..e050371f 100644 --- a/tests/test_choices.py +++ b/tests/test_choices.py @@ -1,4 +1,4 @@ -"""Tests for FileChoices dynamic parameter choices.""" +"""Tests for ChoicesFromFile dynamic parameter choices.""" import tempfile from unittest import TestCase @@ -6,54 +6,54 @@ import pytest from openhexa.sdk.pipelines.exceptions import InvalidParameterError -from openhexa.sdk.pipelines.parameter import FileChoices, Parameter, parameter +from openhexa.sdk.pipelines.parameter import ChoicesFromFile, Parameter, parameter from openhexa.sdk.pipelines.runtime import get_pipeline # --------------------------------------------------------------------------- -# FileChoices construction +# ChoicesFromFile construction # --------------------------------------------------------------------------- -class TestFileChoicesConstruction: +class TestChoicesFromFileConstruction: def test_csv_auto_detected(self): - fc = FileChoices("districts.csv") + fc = ChoicesFromFile("districts.csv") assert fc.format == "csv" assert fc.path == "districts.csv" assert fc.column is None def test_json_auto_detected(self): - fc = FileChoices("data/regions.json", column="code") + fc = ChoicesFromFile("data/regions.json", column="code") assert fc.format == "json" assert fc.column == "code" def test_yaml_auto_detected(self): - assert FileChoices("list.yaml").format == "yaml" + assert ChoicesFromFile("list.yaml").format == "yaml" def test_yml_normalised_to_yaml(self): - assert FileChoices("list.yml").format == "yaml" + assert ChoicesFromFile("list.yml").format == "yaml" def test_unsupported_extension_raises(self): with pytest.raises(InvalidParameterError, match="Supported extensions"): - FileChoices("districts.xlsx") + ChoicesFromFile("districts.xlsx") def test_no_extension_raises(self): with pytest.raises(InvalidParameterError, match="Supported extensions"): - FileChoices("districts") + ChoicesFromFile("districts") def test_empty_path_raises(self): with pytest.raises(InvalidParameterError): - FileChoices("") + ChoicesFromFile("") def test_non_string_column_raises(self): with pytest.raises(InvalidParameterError): - FileChoices("districts.csv", column=42) + ChoicesFromFile("districts.csv", column=42) def test_to_dict(self): - fc = FileChoices("data/districts.csv", column="code") + fc = ChoicesFromFile("data/districts.csv", column="code") assert fc.to_dict() == {"format": "csv", "path": "data/districts.csv", "column": "code"} def test_to_dict_no_column(self): - fc = FileChoices("districts.csv") + fc = ChoicesFromFile("districts.csv") assert fc.to_dict() == {"format": "csv", "path": "districts.csv", "column": None} @@ -62,49 +62,49 @@ def test_to_dict_no_column(self): # --------------------------------------------------------------------------- -class TestParameterWithFileChoices: +class TestParameterWithChoicesFromFile: def test_accepts_file_choices(self): - p = Parameter(code="district", type=str, choices=FileChoices("districts.csv")) - assert isinstance(p.choices, FileChoices) + p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv")) + assert isinstance(p.choices, ChoicesFromFile) def test_to_dict_emits_file_choices_key(self): - p = Parameter(code="district", type=str, choices=FileChoices("districts.csv", column="code")) + p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv", column="code")) d = p.to_dict() assert d["choices"] is None - assert d["file_choices"] == {"format": "csv", "path": "districts.csv", "column": "code"} + assert d["choices_from_file"] == {"format": "csv", "path": "districts.csv", "column": "code"} def test_to_dict_no_file_choices_key_for_static_choices(self): p = Parameter(code="country", type=str, choices=["UG", "KE"]) d = p.to_dict() assert d["choices"] == ["UG", "KE"] - assert "file_choices" not in d + assert "choices_from_file" not in d def test_rejects_file_choices_on_bool_type(self): with pytest.raises(InvalidParameterError, match="don't accept choices"): - Parameter(code="flag", type=bool, choices=FileChoices("flags.csv")) + Parameter(code="flag", type=bool, choices=ChoicesFromFile("flags.csv")) def test_validate_single_skips_choices_check(self): - p = Parameter(code="district", type=str, choices=FileChoices("districts.csv")) + p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv")) # Any string value passes — the platform validates against the resolved list assert p.validate("any_value") == "any_value" def test_validate_multiple_skips_choices_check(self): - p = Parameter(code="district", type=str, choices=FileChoices("districts.csv"), multiple=True) + p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv"), multiple=True) assert p.validate(["A", "B", "C"]) == ["A", "B", "C"] def test_default_not_validated_against_file_choices(self): # Should not raise even though default isn't in any resolved list - p = Parameter(code="district", type=str, choices=FileChoices("districts.csv"), default="UNKNOWN") + p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv"), default="UNKNOWN") assert p.default == "UNKNOWN" def test_decorator_with_file_choices(self): - @parameter(code="district", type=str, choices=FileChoices("districts.csv")) + @parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv")) def my_pipeline(district): pass params = my_pipeline.get_all_parameters() assert len(params) == 1 - assert isinstance(params[0].choices, FileChoices) + assert isinstance(params[0].choices, ChoicesFromFile) # --------------------------------------------------------------------------- @@ -112,14 +112,14 @@ def my_pipeline(district): # --------------------------------------------------------------------------- -class TestAstFileChoices(TestCase): +class TestAstChoicesFromFile(TestCase): def _write_pipeline(self, tmpdir, param_line): with open(f"{tmpdir}/pipeline.py", "w") as f: f.write( "\n".join( [ "from openhexa.sdk.pipelines import pipeline, parameter", - "from openhexa.sdk.pipelines.parameter import FileChoices", + "from openhexa.sdk.pipelines.parameter import ChoicesFromFile", "", param_line, "@pipeline(name='Test pipeline')", @@ -133,42 +133,42 @@ def test_file_choices_csv_positional_path(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, - "@parameter('district', type=str, choices=FileChoices('districts.csv'))", + "@parameter('district', type=str, choices=ChoicesFromFile('districts.csv'))", ) p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] assert param_dict["choices"] is None - assert param_dict["file_choices"] == {"format": "csv", "path": "districts.csv", "column": None} + assert param_dict["choices_from_file"] == {"format": "csv", "path": "districts.csv", "column": None} def test_file_choices_csv_with_column(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, - "@parameter('district', type=str, choices=FileChoices('data/districts.csv', column='code'))", + "@parameter('district', type=str, choices=ChoicesFromFile('data/districts.csv', column='code'))", ) p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] - assert param_dict["file_choices"] == {"format": "csv", "path": "data/districts.csv", "column": "code"} + assert param_dict["choices_from_file"] == {"format": "csv", "path": "data/districts.csv", "column": "code"} def test_file_choices_json(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, - "@parameter('district', type=str, choices=FileChoices('regions.json', column='id'))", + "@parameter('district', type=str, choices=ChoicesFromFile('regions.json', column='id'))", ) p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] - assert param_dict["file_choices"]["format"] == "json" + assert param_dict["choices_from_file"]["format"] == "json" def test_file_choices_yaml(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, - "@parameter('district', type=str, choices=FileChoices('list.yml'))", + "@parameter('district', type=str, choices=ChoicesFromFile('list.yml'))", ) p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] - assert param_dict["file_choices"]["format"] == "yaml" + assert param_dict["choices_from_file"]["format"] == "yaml" def test_unsupported_call_in_choices_raises(self): with tempfile.TemporaryDirectory() as tmpdir: From 9e123f302fce3d2db7727ac21c753f2a8a2ae5bf Mon Sep 17 00:00:00 2001 From: mrivar Date: Mon, 4 May 2026 14:45:28 +0200 Subject: [PATCH 05/18] fix: lint --- Makefile | 3 +++ openhexa/sdk/pipelines/parameter/decorator.py | 6 +++++- openhexa/sdk/pipelines/runtime.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..238042c1 --- /dev/null +++ b/Makefile @@ -0,0 +1,3 @@ +l lint: + @echo "Executing lint in backend code (pre-commit)" + pre-commit run --show-diff-on-failure --color=always --all-files diff --git a/openhexa/sdk/pipelines/parameter/decorator.py b/openhexa/sdk/pipelines/parameter/decorator.py index e90ed9b9..f5a77b83 100644 --- a/openhexa/sdk/pipelines/parameter/decorator.py +++ b/openhexa/sdk/pipelines/parameter/decorator.py @@ -138,7 +138,11 @@ def _validate_single(self, value: typing.Any): return None pre_validated = self.type.validate(normalized_value) - if self.choices is not None and not isinstance(self.choices, ChoicesFromFile) and pre_validated not in self.choices: + if ( + self.choices is not None + and not isinstance(self.choices, ChoicesFromFile) + and pre_validated not in self.choices + ): raise ParameterValueError(f"The provided value for {self.code} is not included in the provided choices.") return pre_validated diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index 399390e1..a1c89335 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -16,8 +16,8 @@ from openhexa.sdk.pipelines.exceptions import InvalidParameterError, PipelineNotFound from openhexa.sdk.pipelines.parameter import ( TYPES_BY_PYTHON_TYPE, - DHIS2Widget, ChoicesFromFile, + DHIS2Widget, IASOWidget, Parameter, validate_parameters, From 5e1f110aff4548b65df44dd0879a5956c58b1800 Mon Sep 17 00:00:00 2001 From: mrivar Date: Mon, 4 May 2026 15:09:07 +0200 Subject: [PATCH 06/18] fix: lint dosctrings --- openhexa/sdk/pipelines/parameter/choices.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py index 56736fac..4f5e2ef6 100644 --- a/openhexa/sdk/pipelines/parameter/choices.py +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -40,12 +40,14 @@ def _detect_format(path: str) -> str: return "yaml" if ext == "yml" else ext def validate_spec(self): + """Validate the path and column specification.""" if not self.path or not isinstance(self.path, str): raise InvalidParameterError("ChoicesFromFile path must be a non-empty string.") if self.column is not None and not isinstance(self.column, str): raise InvalidParameterError("ChoicesFromFile column must be a string.") def to_dict(self) -> dict: + """Return a dictionary representation of the choices spec.""" return { "format": self.format, "path": self.path, From 556bd89cf210c97a3c646d51ff649c73d4aafd21 Mon Sep 17 00:00:00 2001 From: mrivar Date: Tue, 5 May 2026 16:38:07 +0200 Subject: [PATCH 07/18] feat: improve ast for callables --- .../pipelines/parameter/ast_constructible.py | 34 +++++++++++++++++++ openhexa/sdk/pipelines/parameter/choices.py | 4 ++- openhexa/sdk/pipelines/runtime.py | 17 +++++----- tests/test_choices.py | 10 ++++++ 4 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 openhexa/sdk/pipelines/parameter/ast_constructible.py diff --git a/openhexa/sdk/pipelines/parameter/ast_constructible.py b/openhexa/sdk/pipelines/parameter/ast_constructible.py new file mode 100644 index 00000000..1ffbc851 --- /dev/null +++ b/openhexa/sdk/pipelines/parameter/ast_constructible.py @@ -0,0 +1,34 @@ +"""Mixin for classes that can reconstruct themselves from an AST Call node.""" + +import ast +import inspect + + +class AstConstructible: + """Mixin that enables reconstruction of a class instance from an AST Call node. + + Any class whose ``__init__`` takes only scalar (``ast.Constant``) arguments + can inherit from this mixin to gain automatic ``from_ast_call`` support in + the pipeline AST parser — no changes to the parser needed when new parameters + are added to the class. + """ + + @classmethod + def from_ast_call(cls, node: ast.Call) -> "AstConstructible": + """Reconstruct an instance from an AST Call node. + + Maps positional args to ``__init__`` parameter names via + ``inspect.signature``, then merges keyword args, and calls ``cls``. + """ + param_names = list(inspect.signature(cls).parameters.keys()) + kwargs = { + param_names[i]: arg.value + for i, arg in enumerate(node.args) + if isinstance(arg, ast.Constant) and i < len(param_names) + } + kwargs |= { + kw.arg: kw.value.value + for kw in node.keywords + if isinstance(kw.value, ast.Constant) + } + return cls(**kwargs) diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py index 4f5e2ef6..b49577af 100644 --- a/openhexa/sdk/pipelines/parameter/choices.py +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -2,10 +2,12 @@ from openhexa.sdk.pipelines.exceptions import InvalidParameterError +from .ast_constructible import AstConstructible + _SUPPORTED_FORMATS = {"csv", "json", "yaml", "yml"} -class ChoicesFromFile: +class ChoicesFromFile(AstConstructible): """Descriptor for choices loaded dynamically from a file in the workspace file system. The file format is inferred from the path extension (.csv, .json, .yaml, .yml). diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index a1c89335..36ea10f6 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -22,6 +22,12 @@ Parameter, validate_parameters, ) + +# Maps AST function names to classes that support from_ast_call(). +# Add an entry here when introducing a new AstConstructible type. +_AST_CALLABLE_TYPES: dict[str, type] = { + "ChoicesFromFile": ChoicesFromFile, +} from openhexa.sdk.utils import Settings from .pipeline import Pipeline @@ -176,16 +182,9 @@ def _get_decorator_arg_value(decorator: ast.Call, arg: Argument, index: int) -> elif isinstance(keyword.value, ast.Call): func = keyword.value.func func_name = func.id if isinstance(func, ast.Name) else None - if func_name != "ChoicesFromFile": + if func_name not in _AST_CALLABLE_TYPES: raise ValueError(f"Unsupported call in choices argument: {func_name}") - # Extract positional arg (path) and keyword args (column, format override) - pos_args = [a.value for a in keyword.value.args if isinstance(a, ast.Constant)] - kw_args = { - kw.arg: kw.value.value for kw in keyword.value.keywords if isinstance(kw.value, ast.Constant) - } - if pos_args: - kw_args.setdefault("path", pos_args[0]) - return (ChoicesFromFile(**kw_args), True) + return _AST_CALLABLE_TYPES[func_name].from_ast_call(keyword.value), True elif isinstance(keyword.value, ast.Attribute): if keyword.value.attr in DHIS2Widget.__members__: return getattr(DHIS2Widget, keyword.value.attr), True diff --git a/tests/test_choices.py b/tests/test_choices.py index e050371f..55622702 100644 --- a/tests/test_choices.py +++ b/tests/test_choices.py @@ -150,6 +150,16 @@ def test_file_choices_csv_with_column(self): param_dict = p.to_dict()["parameters"][0] assert param_dict["choices_from_file"] == {"format": "csv", "path": "data/districts.csv", "column": "code"} + def test_file_choices_csv_with_column_positional(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices=ChoicesFromFile('data/districts.csv', 'code'))", + ) + p = get_pipeline(tmpdir) + param_dict = p.to_dict()["parameters"][0] + assert param_dict["choices_from_file"] == {"format": "csv", "path": "data/districts.csv", "column": "code"} + def test_file_choices_json(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( From e0323ca596d5897d618cd25504f217ddece9d5f0 Mon Sep 17 00:00:00 2001 From: mrivar Date: Wed, 6 May 2026 15:51:43 +0200 Subject: [PATCH 08/18] feat: add string shorthand choices --- openhexa/sdk/pipelines/parameter/decorator.py | 6 +- openhexa/sdk/pipelines/runtime.py | 13 +- tests/test_choices.py | 190 ++++++++++++++++++ 3 files changed, 204 insertions(+), 5 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/decorator.py b/openhexa/sdk/pipelines/parameter/decorator.py index f5a77b83..05f316ee 100644 --- a/openhexa/sdk/pipelines/parameter/decorator.py +++ b/openhexa/sdk/pipelines/parameter/decorator.py @@ -43,7 +43,7 @@ def __init__( | File ], name: str | None = None, - choices: typing.Sequence | ChoicesFromFile | None = None, + choices: typing.Sequence | ChoicesFromFile | str | None = None, help: str | None = None, default: typing.Any | None = None, widget: DHIS2Widget | IASOWidget | None = None, @@ -67,6 +67,8 @@ def __init__( if choices is not None: if not self.type.accepts_choices: raise InvalidParameterError(f"Parameters of type {self.type} don't accept choices.") + if isinstance(choices, str): + choices = ChoicesFromFile(choices) if isinstance(choices, ChoicesFromFile): # validate_spec() already ran in ChoicesFromFile.__init__; nothing more to check here pass @@ -244,7 +246,7 @@ def parameter( | File ], name: str | None = None, - choices: typing.Sequence | ChoicesFromFile | None = None, + choices: typing.Sequence | ChoicesFromFile | str | None = None, help: str | None = None, widget: DHIS2Widget | IASOWidget | None = None, connection: str | None = None, diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index 36ea10f6..31d015f0 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -8,7 +8,7 @@ import sys from dataclasses import dataclass, field from pathlib import Path -from typing import Any +from typing import Any, Callable from zipfile import ZipFile import requests @@ -37,9 +37,10 @@ class Argument: """Argument of a decorator.""" - name: str # Use str instead of string + name: str types: list[type] = field(default_factory=list) default_value: Any = None + transform: Callable | None = None def import_pipeline(pipeline_dir_path: str) -> Pipeline: @@ -214,6 +215,8 @@ def _get_decorator_spec(decorator: ast.Call, args: tuple[Argument, ...]) -> dict args_spec = {} for i, arg in enumerate(args): value, is_keyword = _get_decorator_arg_value(decorator, arg, i) + if arg.transform is not None: + value = arg.transform(value) args_spec[arg.name] = {"value": value, "is_keyword": is_keyword} return args_spec @@ -300,7 +303,11 @@ def get_pipeline(pipeline_path: Path) -> Pipeline: Argument("code", [ast.Constant]), Argument("type", [ast.Name]), Argument("name", [ast.Constant]), - Argument("choices", [ast.List, ast.Call]), + Argument( + "choices", + [ast.List, ast.Call, ast.Constant], + transform=lambda v: ChoicesFromFile(v) if isinstance(v, str) else v, + ), Argument("help", [ast.Constant]), Argument("default", [ast.Constant, ast.List]), Argument("widget", [ast.Attribute]), diff --git a/tests/test_choices.py b/tests/test_choices.py index 55622702..c23cd0ce 100644 --- a/tests/test_choices.py +++ b/tests/test_choices.py @@ -57,6 +57,196 @@ def test_to_dict_no_column(self): assert fc.to_dict() == {"format": "csv", "path": "districts.csv", "column": None} +# --------------------------------------------------------------------------- +# String shorthand — Parameter.__init__ +# --------------------------------------------------------------------------- + + +class TestStringShorthand: + # --- happy paths --- + + def test_string_shorthand_csv(self): + p = Parameter(code="district", type=str, choices="districts.csv") + assert isinstance(p.choices, ChoicesFromFile) + assert p.choices.path == "districts.csv" + assert p.choices.format == "csv" + assert p.choices.column is None + + def test_string_shorthand_json(self): + p = Parameter(code="district", type=str, choices="data/regions.json") + assert isinstance(p.choices, ChoicesFromFile) + assert p.choices.format == "json" + + def test_string_shorthand_yaml(self): + p = Parameter(code="district", type=str, choices="list.yaml") + assert isinstance(p.choices, ChoicesFromFile) + assert p.choices.format == "yaml" + + def test_string_shorthand_yml(self): + p = Parameter(code="district", type=str, choices="list.yml") + assert isinstance(p.choices, ChoicesFromFile) + assert p.choices.format == "yaml" + + def test_string_shorthand_leading_slash_stripped(self): + p = Parameter(code="district", type=str, choices="/choices.csv") + assert p.choices.path == "/choices.csv" # ChoicesFromFile stores as-is; stripping is app-side + + def test_string_shorthand_serialises_same_as_explicit(self): + shorthand = Parameter(code="district", type=str, choices="districts.csv").to_dict() + explicit = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv")).to_dict() + assert shorthand == explicit + + # --- static list still works --- + + def test_static_list_unaffected(self): + p = Parameter(code="country", type=str, choices=["UG", "KE"]) + assert p.choices == ["UG", "KE"] + + def test_explicit_choices_from_file_unaffected(self): + p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv", column="code")) + assert p.choices.column == "code" + + # --- invalid strings raise clearly --- + + def test_string_no_extension_raises(self): + with pytest.raises(InvalidParameterError, match="Supported extensions"): + Parameter(code="district", type=str, choices="nodot") + + def test_string_unsupported_extension_raises(self): + with pytest.raises(InvalidParameterError, match="Supported extensions"): + Parameter(code="district", type=str, choices="file.xlsx") + + def test_empty_string_raises(self): + with pytest.raises(InvalidParameterError): + Parameter(code="district", type=str, choices="") + + # --- column cannot be specified via shorthand --- + + def test_shorthand_has_no_column(self): + p = Parameter(code="district", type=str, choices="districts.csv") + assert p.choices.column is None + + def test_decorator_with_string_shorthand(self): + @parameter(code="district", type=str, choices="districts.csv") + def my_pipeline(district): + pass + + params = my_pipeline.get_all_parameters() + assert isinstance(params[0].choices, ChoicesFromFile) + + +# --------------------------------------------------------------------------- +# String shorthand — AST round-trip +# --------------------------------------------------------------------------- + + +class TestAstStringShorthand(TestCase): + def _write_pipeline(self, tmpdir, param_line): + with open(f"{tmpdir}/pipeline.py", "w") as f: + f.write( + "\n".join( + [ + "from openhexa.sdk.pipelines import pipeline, parameter", + "", + param_line, + "@pipeline(name='Test pipeline')", + "def test_pipeline(district):", + " pass", + ] + ) + ) + + def test_ast_string_shorthand_csv(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices='districts.csv')", + ) + p = get_pipeline(tmpdir) + param_dict = p.to_dict()["parameters"][0] + assert param_dict["choices"] is None + assert param_dict["choices_from_file"] == {"format": "csv", "path": "districts.csv", "column": None} + + def test_ast_string_shorthand_json(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices='regions.json')", + ) + p = get_pipeline(tmpdir) + assert p.to_dict()["parameters"][0]["choices_from_file"]["format"] == "json" + + def test_ast_string_shorthand_yaml(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices='list.yml')", + ) + p = get_pipeline(tmpdir) + assert p.to_dict()["parameters"][0]["choices_from_file"]["format"] == "yaml" + + def test_ast_string_shorthand_same_output_as_explicit(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices='districts.csv')", + ) + shorthand_dict = get_pipeline(tmpdir).to_dict()["parameters"][0] + + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices=ChoicesFromFile('districts.csv'))", + ) + # need the import for the explicit form + with open(f"{tmpdir}/pipeline.py", "w") as f: + f.write( + "\n".join( + [ + "from openhexa.sdk.pipelines import pipeline, parameter", + "from openhexa.sdk.pipelines.parameter import ChoicesFromFile", + "", + "@parameter('district', type=str, choices=ChoicesFromFile('districts.csv'))", + "@pipeline(name='Test pipeline')", + "def test_pipeline(district):", + " pass", + ] + ) + ) + explicit_dict = get_pipeline(tmpdir).to_dict()["parameters"][0] + + assert shorthand_dict == explicit_dict + + def test_ast_static_list_unaffected(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('country', type=str, choices=['UG', 'KE'])", + ) + p = get_pipeline(tmpdir) + param_dict = p.to_dict()["parameters"][0] + assert param_dict["choices"] == ["UG", "KE"] + assert "choices_from_file" not in param_dict + + def test_ast_string_no_extension_raises(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices='nodot')", + ) + with self.assertRaises(InvalidParameterError): + get_pipeline(tmpdir) + + def test_ast_string_unsupported_extension_raises(self): + with tempfile.TemporaryDirectory() as tmpdir: + self._write_pipeline( + tmpdir, + "@parameter('district', type=str, choices='file.xlsx')", + ) + with self.assertRaises(InvalidParameterError): + get_pipeline(tmpdir) + + # --------------------------------------------------------------------------- # Parameter integration # --------------------------------------------------------------------------- From 13a00419e79f07ba7704eb540920e83964266e9f Mon Sep 17 00:00:00 2001 From: mrivar Date: Wed, 6 May 2026 15:52:15 +0200 Subject: [PATCH 09/18] fix: lint --- openhexa/sdk/pipelines/parameter/ast_constructible.py | 6 +----- openhexa/sdk/pipelines/runtime.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/ast_constructible.py b/openhexa/sdk/pipelines/parameter/ast_constructible.py index 1ffbc851..6ce820bf 100644 --- a/openhexa/sdk/pipelines/parameter/ast_constructible.py +++ b/openhexa/sdk/pipelines/parameter/ast_constructible.py @@ -26,9 +26,5 @@ def from_ast_call(cls, node: ast.Call) -> "AstConstructible": for i, arg in enumerate(node.args) if isinstance(arg, ast.Constant) and i < len(param_names) } - kwargs |= { - kw.arg: kw.value.value - for kw in node.keywords - if isinstance(kw.value, ast.Constant) - } + kwargs |= {kw.arg: kw.value.value for kw in node.keywords if isinstance(kw.value, ast.Constant)} return cls(**kwargs) diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index 31d015f0..ef360ce8 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -6,9 +6,10 @@ import io import os import sys +from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Callable +from typing import Any from zipfile import ZipFile import requests From 4f97ab0825cfdc43f02d64763cc55e668c618c4a Mon Sep 17 00:00:00 2001 From: mrivar Date: Wed, 6 May 2026 16:22:16 +0200 Subject: [PATCH 10/18] feat: minor improvements --- .../pipelines/parameter/ast_constructible.py | 32 +++++++++++++------ openhexa/sdk/pipelines/parameter/choices.py | 17 ++++++++-- openhexa/sdk/pipelines/parameter/decorator.py | 5 +-- openhexa/sdk/pipelines/runtime.py | 6 ++-- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/ast_constructible.py b/openhexa/sdk/pipelines/parameter/ast_constructible.py index 6ce820bf..67f7c67c 100644 --- a/openhexa/sdk/pipelines/parameter/ast_constructible.py +++ b/openhexa/sdk/pipelines/parameter/ast_constructible.py @@ -8,9 +8,14 @@ class AstConstructible: """Mixin that enables reconstruction of a class instance from an AST Call node. Any class whose ``__init__`` takes only scalar (``ast.Constant``) arguments - can inherit from this mixin to gain automatic ``from_ast_call`` support in - the pipeline AST parser — no changes to the parser needed when new parameters - are added to the class. + can inherit from this mixin and get ``from_ast_call`` for free. Adding or + renaming ``__init__`` parameters does *not* require touching the parser. + + To make the AST parser recognise a new subclass by name, add one entry to + ``_AST_CALLABLE_TYPES`` in ``runtime.py`` (and ensure the subclass module is + imported there). Auto-registration via ``__init_subclass__`` would not remove + that requirement — the registry entry only exists after the module is imported, + so an explicit import would still be needed. """ @classmethod @@ -21,10 +26,19 @@ def from_ast_call(cls, node: ast.Call) -> "AstConstructible": ``inspect.signature``, then merges keyword args, and calls ``cls``. """ param_names = list(inspect.signature(cls).parameters.keys()) - kwargs = { - param_names[i]: arg.value - for i, arg in enumerate(node.args) - if isinstance(arg, ast.Constant) and i < len(param_names) - } - kwargs |= {kw.arg: kw.value.value for kw in node.keywords if isinstance(kw.value, ast.Constant)} + kwargs = {} + for i, arg in enumerate(node.args): + if i >= len(param_names): + break + if not isinstance(arg, ast.Constant): + raise ValueError( + f"{cls.__name__}() positional argument {i + 1} must be a literal value, not a dynamic expression." + ) + kwargs[param_names[i]] = arg.value + for kw in node.keywords: + if not isinstance(kw.value, ast.Constant): + raise ValueError( + f"{cls.__name__}() keyword argument '{kw.arg}' must be a literal value, not a dynamic expression." + ) + kwargs[kw.arg] = kw.value.value return cls(**kwargs) diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py index b49577af..c37b5026 100644 --- a/openhexa/sdk/pipelines/parameter/choices.py +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -26,13 +26,11 @@ class ChoicesFromFile(AstConstructible): def __init__(self, path: str, column: str | None = None): self.path = path self.column = column - self.format = self._detect_format(path) self.validate_spec() + self.format = self._detect_format(path) @staticmethod def _detect_format(path: str) -> str: - if not path or not isinstance(path, str): - raise InvalidParameterError("ChoicesFromFile path must be a non-empty string.") ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" if ext not in _SUPPORTED_FORMATS: raise InvalidParameterError( @@ -48,6 +46,19 @@ def validate_spec(self): if self.column is not None and not isinstance(self.column, str): raise InvalidParameterError("ChoicesFromFile column must be a string.") + def __repr__(self) -> str: + if self.column is not None: + return f"ChoicesFromFile({self.path!r}, column={self.column!r})" + return f"ChoicesFromFile({self.path!r})" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ChoicesFromFile): + return NotImplemented + return self.path == other.path and self.column == other.column + + def __hash__(self) -> int: + return hash((self.path, self.column)) + def to_dict(self) -> dict: """Return a dictionary representation of the choices spec.""" return { diff --git a/openhexa/sdk/pipelines/parameter/decorator.py b/openhexa/sdk/pipelines/parameter/decorator.py index 05f316ee..2697b7ec 100644 --- a/openhexa/sdk/pipelines/parameter/decorator.py +++ b/openhexa/sdk/pipelines/parameter/decorator.py @@ -69,10 +69,7 @@ def __init__( raise InvalidParameterError(f"Parameters of type {self.type} don't accept choices.") if isinstance(choices, str): choices = ChoicesFromFile(choices) - if isinstance(choices, ChoicesFromFile): - # validate_spec() already ran in ChoicesFromFile.__init__; nothing more to check here - pass - else: + elif not isinstance(choices, ChoicesFromFile): if len(choices) == 0: raise InvalidParameterError("Choices, if provided, cannot be empty.") try: diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index ef360ce8..72a97093 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -23,15 +23,15 @@ Parameter, validate_parameters, ) +from openhexa.sdk.utils import Settings + +from .pipeline import Pipeline # Maps AST function names to classes that support from_ast_call(). # Add an entry here when introducing a new AstConstructible type. _AST_CALLABLE_TYPES: dict[str, type] = { "ChoicesFromFile": ChoicesFromFile, } -from openhexa.sdk.utils import Settings - -from .pipeline import Pipeline @dataclass From 08b13e54b39c2e2071ca299a2bf6b6683d4bd994 Mon Sep 17 00:00:00 2001 From: mrivar Date: Wed, 6 May 2026 16:45:29 +0200 Subject: [PATCH 11/18] feat: minor minor improvements --- openhexa/sdk/pipelines/parameter/choices.py | 4 ++-- openhexa/sdk/pipelines/parameter/decorator.py | 2 +- tests/test_choices.py | 9 +++------ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py index c37b5026..a8c6aeb8 100644 --- a/openhexa/sdk/pipelines/parameter/choices.py +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -26,7 +26,7 @@ class ChoicesFromFile(AstConstructible): def __init__(self, path: str, column: str | None = None): self.path = path self.column = column - self.validate_spec() + self._validate_spec() self.format = self._detect_format(path) @staticmethod @@ -39,7 +39,7 @@ def _detect_format(path: str) -> str: ) return "yaml" if ext == "yml" else ext - def validate_spec(self): + def _validate_spec(self): """Validate the path and column specification.""" if not self.path or not isinstance(self.path, str): raise InvalidParameterError("ChoicesFromFile path must be a non-empty string.") diff --git a/openhexa/sdk/pipelines/parameter/decorator.py b/openhexa/sdk/pipelines/parameter/decorator.py index 2697b7ec..01050fb4 100644 --- a/openhexa/sdk/pipelines/parameter/decorator.py +++ b/openhexa/sdk/pipelines/parameter/decorator.py @@ -277,7 +277,7 @@ def parameter( An optional default value for the parameter (should be of the type defined by the type parameter) required : bool, default=True Whether the parameter is mandatory - multiple : bool, default=True + multiple : bool, default=False Whether this parameter should be provided multiple values (if True, the value must be provided as a list of values of the chosen type) directory : str, optional diff --git a/tests/test_choices.py b/tests/test_choices.py index c23cd0ce..3fddcc70 100644 --- a/tests/test_choices.py +++ b/tests/test_choices.py @@ -67,10 +67,7 @@ class TestStringShorthand: def test_string_shorthand_csv(self): p = Parameter(code="district", type=str, choices="districts.csv") - assert isinstance(p.choices, ChoicesFromFile) - assert p.choices.path == "districts.csv" - assert p.choices.format == "csv" - assert p.choices.column is None + assert p.choices == ChoicesFromFile("districts.csv") def test_string_shorthand_json(self): p = Parameter(code="district", type=str, choices="data/regions.json") @@ -104,7 +101,7 @@ def test_static_list_unaffected(self): def test_explicit_choices_from_file_unaffected(self): p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv", column="code")) - assert p.choices.column == "code" + assert p.choices == ChoicesFromFile("districts.csv", column="code") # --- invalid strings raise clearly --- @@ -124,7 +121,7 @@ def test_empty_string_raises(self): def test_shorthand_has_no_column(self): p = Parameter(code="district", type=str, choices="districts.csv") - assert p.choices.column is None + assert p.choices == ChoicesFromFile("districts.csv") def test_decorator_with_string_shorthand(self): @parameter(code="district", type=str, choices="districts.csv") From 9cfaa5c4294ec302fb826cc1d58837962c3b2bac Mon Sep 17 00:00:00 2001 From: mrivar Date: Wed, 6 May 2026 16:53:14 +0200 Subject: [PATCH 12/18] feat: add docstrings --- openhexa/sdk/pipelines/parameter/choices.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py index a8c6aeb8..00e0eadf 100644 --- a/openhexa/sdk/pipelines/parameter/choices.py +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -47,16 +47,19 @@ def _validate_spec(self): raise InvalidParameterError("ChoicesFromFile column must be a string.") def __repr__(self) -> str: + """Return a string representation of the ChoicesFromFile instance.""" if self.column is not None: return f"ChoicesFromFile({self.path!r}, column={self.column!r})" return f"ChoicesFromFile({self.path!r})" def __eq__(self, other: object) -> bool: + """Check equality based on path and column.""" if not isinstance(other, ChoicesFromFile): return NotImplemented return self.path == other.path and self.column == other.column def __hash__(self) -> int: + """Return hash based on path and column.""" return hash((self.path, self.column)) def to_dict(self) -> dict: From 71fd0c3b685fce6737ec791a8d48d78ec2a9e5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Riva?= Date: Wed, 6 May 2026 18:27:01 +0200 Subject: [PATCH 13/18] recover comment --- openhexa/sdk/pipelines/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index 72a97093..bce7c10b 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -38,7 +38,7 @@ class Argument: """Argument of a decorator.""" - name: str + name: str # Use str instead of string types: list[type] = field(default_factory=list) default_value: Any = None transform: Callable | None = None From c39a2136dd15e4b821a1fce015a3fd01f2f4b248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Riva?= Date: Wed, 6 May 2026 18:27:37 +0200 Subject: [PATCH 14/18] typo --- openhexa/sdk/pipelines/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openhexa/sdk/pipelines/runtime.py b/openhexa/sdk/pipelines/runtime.py index bce7c10b..e23888a6 100644 --- a/openhexa/sdk/pipelines/runtime.py +++ b/openhexa/sdk/pipelines/runtime.py @@ -38,7 +38,7 @@ class Argument: """Argument of a decorator.""" - name: str # Use str instead of string + name: str # Use str instead of string types: list[type] = field(default_factory=list) default_value: Any = None transform: Callable | None = None From 1066079cbc344940d73f604ee87fbb1ada16fc3f Mon Sep 17 00:00:00 2001 From: mrivar Date: Mon, 11 May 2026 11:47:55 +0200 Subject: [PATCH 15/18] fix: conflict --- openhexa/sdk/pipelines/parameter/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/__init__.py b/openhexa/sdk/pipelines/parameter/__init__.py index 38ff31e7..774fd1ec 100644 --- a/openhexa/sdk/pipelines/parameter/__init__.py +++ b/openhexa/sdk/pipelines/parameter/__init__.py @@ -5,10 +5,7 @@ from openhexa.sdk.pipelines.exceptions import InvalidParameterError, ParameterValueError -<<<<<<< HEAD from .choices import ChoicesFromFile -======= ->>>>>>> HEXA-1620-parameters-refactor from .decorator import FunctionWithParameter, Parameter, parameter, validate_parameters from .types import ( TYPES_BY_PYTHON_TYPE, From 85ad208d66e8f5695954da9215b5d62aa9d1d8ff Mon Sep 17 00:00:00 2001 From: mrivar Date: Thu, 14 May 2026 20:44:04 +0200 Subject: [PATCH 16/18] feat: remove auto detect format --- openhexa/sdk/pipelines/parameter/choices.py | 42 ++++++++++----------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py index 00e0eadf..0eaafb00 100644 --- a/openhexa/sdk/pipelines/parameter/choices.py +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -4,16 +4,12 @@ from .ast_constructible import AstConstructible -_SUPPORTED_FORMATS = {"csv", "json", "yaml", "yml"} +_SUPPORTED_FORMATS = {"csv", "json", "yaml"} class ChoicesFromFile(AstConstructible): """Descriptor for choices loaded dynamically from a file in the workspace file system. - The file format is inferred from the path extension (.csv, .json, .yaml, .yml). - For CSV files with a single column, that column is used automatically. - For CSV/JSON/YAML files with multiple columns/keys, `column` must be specified. - Parameters ---------- path : str @@ -21,23 +17,15 @@ class ChoicesFromFile(AstConstructible): column : str, optional Column name (CSV) or key (JSON/YAML) to use as choice values. Required when the file has more than one column/key. + format : str, optional + File format (e.g. "csv", "json", "yaml"). Sent as-is to the platform. """ - def __init__(self, path: str, column: str | None = None): + def __init__(self, path: str, column: str | None = None, format: str | None = None): self.path = path self.column = column + self.format = format self._validate_spec() - self.format = self._detect_format(path) - - @staticmethod - def _detect_format(path: str) -> str: - ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" - if ext not in _SUPPORTED_FORMATS: - raise InvalidParameterError( - f"Cannot determine file format from path '{path}'. " - f"Supported extensions: {', '.join(sorted(_SUPPORTED_FORMATS))}." - ) - return "yaml" if ext == "yml" else ext def _validate_spec(self): """Validate the path and column specification.""" @@ -45,22 +33,30 @@ def _validate_spec(self): raise InvalidParameterError("ChoicesFromFile path must be a non-empty string.") if self.column is not None and not isinstance(self.column, str): raise InvalidParameterError("ChoicesFromFile column must be a string.") + if self.format is not None and self.format not in _SUPPORTED_FORMATS: + raise InvalidParameterError( + f"ChoicesFromFile format '{self.format}' is not supported. " + f"Supported formats: {', '.join(sorted(_SUPPORTED_FORMATS))}." + ) def __repr__(self) -> str: """Return a string representation of the ChoicesFromFile instance.""" + parts = [repr(self.path)] if self.column is not None: - return f"ChoicesFromFile({self.path!r}, column={self.column!r})" - return f"ChoicesFromFile({self.path!r})" + parts.append(f"column={self.column!r}") + if self.format is not None: + parts.append(f"format={self.format!r}") + return f"ChoicesFromFile({', '.join(parts)})" def __eq__(self, other: object) -> bool: - """Check equality based on path and column.""" + """Check equality based on path, column, and format.""" if not isinstance(other, ChoicesFromFile): return NotImplemented - return self.path == other.path and self.column == other.column + return self.path == other.path and self.column == other.column and self.format == other.format def __hash__(self) -> int: - """Return hash based on path and column.""" - return hash((self.path, self.column)) + """Return hash based on path, column, and format.""" + return hash((self.path, self.column, self.format)) def to_dict(self) -> dict: """Return a dictionary representation of the choices spec.""" From 3807a30a77788649c1d8cbc5cfde5c44fe90750f Mon Sep 17 00:00:00 2001 From: mrivar Date: Thu, 14 May 2026 21:00:23 +0200 Subject: [PATCH 17/18] tests: fix tests --- openhexa/sdk/pipelines/parameter/choices.py | 2 +- tests/test_choices.py | 102 +++++++++++--------- 2 files changed, 55 insertions(+), 49 deletions(-) diff --git a/openhexa/sdk/pipelines/parameter/choices.py b/openhexa/sdk/pipelines/parameter/choices.py index 0eaafb00..ddacc635 100644 --- a/openhexa/sdk/pipelines/parameter/choices.py +++ b/openhexa/sdk/pipelines/parameter/choices.py @@ -4,7 +4,7 @@ from .ast_constructible import AstConstructible -_SUPPORTED_FORMATS = {"csv", "json", "yaml"} +_SUPPORTED_FORMATS = {"csv", "json", "yaml", "yml"} class ChoicesFromFile(AstConstructible): diff --git a/tests/test_choices.py b/tests/test_choices.py index 3fddcc70..a816ba85 100644 --- a/tests/test_choices.py +++ b/tests/test_choices.py @@ -15,30 +15,34 @@ class TestChoicesFromFileConstruction: - def test_csv_auto_detected(self): + def test_format_defaults_to_none(self): fc = ChoicesFromFile("districts.csv") - assert fc.format == "csv" + assert fc.format is None assert fc.path == "districts.csv" assert fc.column is None - def test_json_auto_detected(self): - fc = ChoicesFromFile("data/regions.json", column="code") + def test_explicit_format_accepted(self): + fc = ChoicesFromFile("data/regions.json", column="code", format="json") assert fc.format == "json" assert fc.column == "code" - def test_yaml_auto_detected(self): - assert ChoicesFromFile("list.yaml").format == "yaml" + def test_explicit_format_yaml(self): + assert ChoicesFromFile("list.yaml", format="yaml").format == "yaml" - def test_yml_normalised_to_yaml(self): - assert ChoicesFromFile("list.yml").format == "yaml" + def test_yml_explicit_format_accepted(self): + assert ChoicesFromFile("list.yml", format="yml").format == "yml" - def test_unsupported_extension_raises(self): - with pytest.raises(InvalidParameterError, match="Supported extensions"): - ChoicesFromFile("districts.xlsx") + def test_invalid_explicit_format_raises(self): + with pytest.raises(InvalidParameterError, match="Supported formats"): + ChoicesFromFile("districts.csv", format="excel") - def test_no_extension_raises(self): - with pytest.raises(InvalidParameterError, match="Supported extensions"): - ChoicesFromFile("districts") + def test_any_extension_accepted(self): + fc = ChoicesFromFile("districts.xlsx") + assert fc.format is None + + def test_no_extension_accepted(self): + fc = ChoicesFromFile("districts") + assert fc.format is None def test_empty_path_raises(self): with pytest.raises(InvalidParameterError): @@ -49,12 +53,12 @@ def test_non_string_column_raises(self): ChoicesFromFile("districts.csv", column=42) def test_to_dict(self): - fc = ChoicesFromFile("data/districts.csv", column="code") + fc = ChoicesFromFile("data/districts.csv", column="code", format="csv") assert fc.to_dict() == {"format": "csv", "path": "data/districts.csv", "column": "code"} def test_to_dict_no_column(self): fc = ChoicesFromFile("districts.csv") - assert fc.to_dict() == {"format": "csv", "path": "districts.csv", "column": None} + assert fc.to_dict() == {"format": None, "path": "districts.csv", "column": None} # --------------------------------------------------------------------------- @@ -72,17 +76,17 @@ def test_string_shorthand_csv(self): def test_string_shorthand_json(self): p = Parameter(code="district", type=str, choices="data/regions.json") assert isinstance(p.choices, ChoicesFromFile) - assert p.choices.format == "json" + assert p.choices.format is None def test_string_shorthand_yaml(self): p = Parameter(code="district", type=str, choices="list.yaml") assert isinstance(p.choices, ChoicesFromFile) - assert p.choices.format == "yaml" + assert p.choices.format is None - def test_string_shorthand_yml(self): + def test_string_shorthand_any_extension(self): p = Parameter(code="district", type=str, choices="list.yml") assert isinstance(p.choices, ChoicesFromFile) - assert p.choices.format == "yaml" + assert p.choices.format is None def test_string_shorthand_leading_slash_stripped(self): p = Parameter(code="district", type=str, choices="/choices.csv") @@ -103,15 +107,17 @@ def test_explicit_choices_from_file_unaffected(self): p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv", column="code")) assert p.choices == ChoicesFromFile("districts.csv", column="code") - # --- invalid strings raise clearly --- + # --- any string path is accepted (format defaults to None) --- - def test_string_no_extension_raises(self): - with pytest.raises(InvalidParameterError, match="Supported extensions"): - Parameter(code="district", type=str, choices="nodot") + def test_string_no_extension_accepted(self): + p = Parameter(code="district", type=str, choices="nodot") + assert isinstance(p.choices, ChoicesFromFile) + assert p.choices.format is None - def test_string_unsupported_extension_raises(self): - with pytest.raises(InvalidParameterError, match="Supported extensions"): - Parameter(code="district", type=str, choices="file.xlsx") + def test_string_any_extension_accepted(self): + p = Parameter(code="district", type=str, choices="file.xlsx") + assert isinstance(p.choices, ChoicesFromFile) + assert p.choices.format is None def test_empty_string_raises(self): with pytest.raises(InvalidParameterError): @@ -162,7 +168,7 @@ def test_ast_string_shorthand_csv(self): p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] assert param_dict["choices"] is None - assert param_dict["choices_from_file"] == {"format": "csv", "path": "districts.csv", "column": None} + assert param_dict["choices_from_file"] == {"format": None, "path": "districts.csv", "column": None} def test_ast_string_shorthand_json(self): with tempfile.TemporaryDirectory() as tmpdir: @@ -171,16 +177,16 @@ def test_ast_string_shorthand_json(self): "@parameter('district', type=str, choices='regions.json')", ) p = get_pipeline(tmpdir) - assert p.to_dict()["parameters"][0]["choices_from_file"]["format"] == "json" + assert p.to_dict()["parameters"][0]["choices_from_file"]["format"] is None - def test_ast_string_shorthand_yaml(self): + def test_ast_string_shorthand_any_extension(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, "@parameter('district', type=str, choices='list.yml')", ) p = get_pipeline(tmpdir) - assert p.to_dict()["parameters"][0]["choices_from_file"]["format"] == "yaml" + assert p.to_dict()["parameters"][0]["choices_from_file"]["format"] is None def test_ast_string_shorthand_same_output_as_explicit(self): with tempfile.TemporaryDirectory() as tmpdir: @@ -225,23 +231,23 @@ def test_ast_static_list_unaffected(self): assert param_dict["choices"] == ["UG", "KE"] assert "choices_from_file" not in param_dict - def test_ast_string_no_extension_raises(self): + def test_ast_string_no_extension_accepted(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, "@parameter('district', type=str, choices='nodot')", ) - with self.assertRaises(InvalidParameterError): - get_pipeline(tmpdir) + p = get_pipeline(tmpdir) + assert p.to_dict()["parameters"][0]["choices_from_file"]["format"] is None - def test_ast_string_unsupported_extension_raises(self): + def test_ast_string_any_extension_accepted(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, "@parameter('district', type=str, choices='file.xlsx')", ) - with self.assertRaises(InvalidParameterError): - get_pipeline(tmpdir) + p = get_pipeline(tmpdir) + assert p.to_dict()["parameters"][0]["choices_from_file"]["format"] is None # --------------------------------------------------------------------------- @@ -255,7 +261,7 @@ def test_accepts_file_choices(self): assert isinstance(p.choices, ChoicesFromFile) def test_to_dict_emits_file_choices_key(self): - p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv", column="code")) + p = Parameter(code="district", type=str, choices=ChoicesFromFile("districts.csv", column="code", format="csv")) d = p.to_dict() assert d["choices"] is None assert d["choices_from_file"] == {"format": "csv", "path": "districts.csv", "column": "code"} @@ -316,7 +322,7 @@ def _write_pipeline(self, tmpdir, param_line): ) ) - def test_file_choices_csv_positional_path(self): + def test_file_choices_positional_path(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, @@ -325,9 +331,9 @@ def test_file_choices_csv_positional_path(self): p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] assert param_dict["choices"] is None - assert param_dict["choices_from_file"] == {"format": "csv", "path": "districts.csv", "column": None} + assert param_dict["choices_from_file"] == {"format": None, "path": "districts.csv", "column": None} - def test_file_choices_csv_with_column(self): + def test_file_choices_with_column(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, @@ -335,9 +341,9 @@ def test_file_choices_csv_with_column(self): ) p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] - assert param_dict["choices_from_file"] == {"format": "csv", "path": "data/districts.csv", "column": "code"} + assert param_dict["choices_from_file"] == {"format": None, "path": "data/districts.csv", "column": "code"} - def test_file_choices_csv_with_column_positional(self): + def test_file_choices_with_column_positional(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, @@ -345,19 +351,19 @@ def test_file_choices_csv_with_column_positional(self): ) p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] - assert param_dict["choices_from_file"] == {"format": "csv", "path": "data/districts.csv", "column": "code"} + assert param_dict["choices_from_file"] == {"format": None, "path": "data/districts.csv", "column": "code"} - def test_file_choices_json(self): + def test_file_choices_explicit_format(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, - "@parameter('district', type=str, choices=ChoicesFromFile('regions.json', column='id'))", + "@parameter('district', type=str, choices=ChoicesFromFile('regions.json', column='id', format='json'))", ) p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] assert param_dict["choices_from_file"]["format"] == "json" - def test_file_choices_yaml(self): + def test_file_choices_format_none_by_default(self): with tempfile.TemporaryDirectory() as tmpdir: self._write_pipeline( tmpdir, @@ -365,7 +371,7 @@ def test_file_choices_yaml(self): ) p = get_pipeline(tmpdir) param_dict = p.to_dict()["parameters"][0] - assert param_dict["choices_from_file"]["format"] == "yaml" + assert param_dict["choices_from_file"]["format"] is None def test_unsupported_call_in_choices_raises(self): with tempfile.TemporaryDirectory() as tmpdir: From 16f56afc3e592bee8e95c5e258c5f96739c70006 Mon Sep 17 00:00:00 2001 From: mrivar Date: Thu, 14 May 2026 21:14:12 +0200 Subject: [PATCH 18/18] fix: conda --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d6e9ccbd..40085654 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ requires-python = ">=3.11,<3.15" # the main constraint for supported Python vers dependencies = [ "urllib3<3", "multiprocess~=0.70.15", - "requests>=2.31,<2.34", + "requests>=2.31,<3", "PyYAML~=6.0", "click~=8.1.3", "jinja2>3,<4",