From 1f64a4cc3f24fb73a4160be8d2990883432e5132 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Tue, 16 Jun 2026 12:20:25 +0200 Subject: [PATCH 01/13] CVS-169692_enable_on_commit_tests_part_II --- ci/build_test_OnCommit.groovy | 8 +- tests/functional/config.py | 40 +- tests/functional/conftest.py | 393 ++++++++++++++++++ tests/functional/constants/os_version.py | 42 ++ tests/functional/constants/ovms_images.py | 12 +- tests/functional/constants/paths.py | 11 + tests/functional/utils/docker.py | 2 + tests/functional/utils/download.py | 58 +++ tests/functional/utils/hooks.py | 245 ++++++++++- .../utils/ovms_binary_image/Dockerfile.redhat | 47 +++ .../utils/ovms_binary_image/Dockerfile.ubuntu | 46 ++ .../utils/ovms_capi_image/Dockerfile.redhat | 47 +++ .../utils/ovms_capi_image/Dockerfile.ubuntu | 44 ++ .../ovms_testing_image/Dockerfile.redhat | 2 +- 14 files changed, 951 insertions(+), 46 deletions(-) create mode 100644 tests/functional/conftest.py create mode 100644 tests/functional/constants/os_version.py create mode 100644 tests/functional/utils/download.py create mode 100644 tests/functional/utils/ovms_binary_image/Dockerfile.redhat create mode 100644 tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu create mode 100644 tests/functional/utils/ovms_capi_image/Dockerfile.redhat create mode 100644 tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu diff --git a/ci/build_test_OnCommit.groovy b/ci/build_test_OnCommit.groovy index 163169e1b7..4a7ef4ce01 100644 --- a/ci/build_test_OnCommit.groovy +++ b/ci/build_test_OnCommit.groovy @@ -22,13 +22,15 @@ def validation_branch = "develop" // Override agent node: // [test_agent_linux=] - Run Linux doc tests on (default: ovms_ptl) // [test_agent_windows=] - Run Windows doc tests on (default: ovms_win_ptl) +// example: [test_agent_windows=ovms_win_ptl] // // Override file list (space-separated, converted to pytest -k filter joined with ' or '): // [test_doc_files_linux=] - Use instead of auto-detected list (Linux) // [test_doc_files_windows=] - Use instead of auto-detected list (Windows) +// example: [test_doc_files_linux=demos/continuous_batching/README.md demos/audio/README.md] // // Override validation branch: - // [validation_branch=] - Use instead of default 'develop' for test repo checkout +// [validation_branch=] - Use instead of default 'develop' for test repo checkout // pipeline { @@ -327,7 +329,7 @@ pipeline { def test_doc_files_str = test_doc_files_linux.split('\n').join(' or ') sh "make create-venv && rm -f tests/functional && ln -s ${pwd}/../tests/functional tests/functional" def cmd_venv_activate = ". .venv/bin/activate" - def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500" + def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_LOGGING_LEVEL_OVMS=DEBUG && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500" def cmd_pytest = "pytest tests/non_functional/documentation -k '${test_doc_files_str}' -n 0 --dist loadgroup" def cmd = "" if ( image_build_needed == "true" ) { @@ -392,7 +394,7 @@ pipeline { def ovms_c_repo_path = bat(returnStdout: true, script: 'cd .. && cd').trim().split('\n').last().trim() def cmd_link_ovms = "(if exist ${current_path}\\tests\\functional rmdir ${current_path}\\tests\\functional) && mklink /D ${current_path}\\tests\\functional ${ovms_c_repo_path}\\tests\\functional" def cmd_requirements = "(if not exist .venv virtualenv .venv --python=python3.12) && call .venv\\Scripts\\activate.bat && pip install -r requirements.txt" - def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\"" + def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && \"TT_LOGGING_LEVEL_OVMS=DEBUG\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\"" def cmd_pytest = "pytest tests/non_functional/documentation -k \"${test_doc_files_str}\" -n 0 --dist loadgroup --basetemp=\"C:\\tmp\\pytest-${BRANCH_NAME}-${BUILD_NUMBER}\"" def cmd = "" if ( win_image_build_needed == "true" ) { diff --git a/tests/functional/config.py b/tests/functional/config.py index 47bed51e7a..95abe9772c 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -73,6 +73,11 @@ def get_uses_mapping(): """ TT_OVMS_C_REPO_PATH - path to ovms-c repository. Can be relative or absolute. """ ovms_c_repo_path = get_path("TT_OVMS_C_REPO_PATH", get_path("PWD", "./")) +""" TT_SETUPVARS_SCRIPT_PATH - path to setupvars.bat script """ +setupvars_script_path = os.environ.get( + "TT_SETUPVARS_SCRIPT_PATH", os.path.join(ovms_c_repo_path, "setupvars.bat") +) + """BUILD_LOGS - path to dir where artifacts should be stored""" artifacts_dir = get_path("BUILD_LOGS", os.path.join(ovms_c_repo_path, "tests", "functional", "test_log_build")) @@ -147,18 +152,6 @@ def get_uses_mapping(): """ TT_TARGET_DEVICE - list of devices separated by a comma "CPU,GPU,NPU" """ target_devices = get_target_devices() -target_device = target_devices[0] - -"""IMAGE - docker image name which should be used to run tests""" -if target_device == TargetDevice.GPU: - _default_image = "openvino/model_server-gpu" -else: - _default_image = "openvino/model_server" -image = os.environ.get("IMAGE", _default_image) - -start_minio_container_command = 'server --address ":{}" /data' - -container_minio_log_line = "Console endpoint is listening on a dynamic port" # Reservation manager values, for details study tests.functional.utils.reservation_manager """ TT_GRPC_OVMS_STARTING_PORT - Grpc port where ovms should be exposed""" @@ -171,29 +164,6 @@ def get_uses_mapping(): ports_pool_size = get_int("TT_PORTS_POOL_SIZE", None) # NOTE: Above values will be validated and could be changed if invalid -""" TT_CONVERTED_MODELS_EXPIRE_TIME - Time after converted models are not up-to-date and needs to be refreshed(s) """ -converted_models_expire_time = get_int("TT_CONVERTED_MODELS_EXPIRE_TIME", 7*24*3600) # Set default to one week - -""" TT_DEFAULT_INFER_TIMEOUT - Timeout for CPU target device""" -default_infer_timeout = get_int("TT_DEFAULT_INFER_TIMEOUT", 10) - -""" TT_DEFAULT_GPU_INFER_TIMEOUT - Timeout for GPU target device""" -default_gpu_infer_timeout = get_int("TT_DEFAULT_GPU_INFER_TIMEOUT", 10*default_infer_timeout) - -""" TT_DEFAULT_NPU_INFER_TIMEOUT - Timeout for NPU target device""" -default_npu_infer_timeout = get_int("TT_DEFAULT_NPU_INFER_TIMEOUT", 10*default_infer_timeout) - -""" INFER TIMEOUT """ -infer_timeouts = { - TargetDevice.CPU: default_infer_timeout, - TargetDevice.GPU: default_gpu_infer_timeout, - TargetDevice.NPU: default_npu_infer_timeout, - TargetDevice.AUTO: default_gpu_infer_timeout, - TargetDevice.HETERO: default_gpu_infer_timeout, - TargetDevice.AUTO_CPU_GPU: default_gpu_infer_timeout, -} -infer_timeout = infer_timeouts[target_device] - """ TT_IS_NGINX_MTLS - Specify if given image is OVSA nginx mtls image. """ is_nginx_mtls = get_bool("TT_IS_NGINX_MTLS", False) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 0000000000..a910c29c3d --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,393 @@ +# +# Copyright (c) 2018-2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +import sys +import time +from logging import FileHandler + +import pytest +from _pytest._code import ExceptionInfo, filter_traceback # noqa +from _pytest.outcomes import OutcomeException + +from tests.functional.config import test_dir, test_dir_cleanup, artifacts_dir, enable_pytest_plugins +from tests.functional.constants.components import OvmsComponents +from tests.functional.constants.os_type import OsType +from tests.functional.constants.ovms import ( + BASE_OS_PARAM_NAME, + OVMS_TYPE_PARAM_NAME, + TARGET_DEVICE_PARAM_NAME, + USES_MAPPING_PARAM_NAME, +) +from tests.functional.constants.ovms_binaries import calculate_ovms_binary_name +from tests.functional.constants.ovms_images import calculate_ovms_image_name +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.target_device import TargetDevice +from tests.functional.utils import hooks +from tests.functional.utils.hooks import ( + log_configuration_variables, + parametrize_all_models, + parametrize_base_os, + parametrize_input_shape, + parametrize_iteration_info, + parametrize_many_models, + parametrize_model_aux_type, + parametrize_model_type, + parametrize_ovms_type, + parametrize_plugin_config, + parametrize_target_device, + parametrize_uses_mapping, + validate_port_pool, +) +from tests.functional.utils.marks import MarksRegistry, MarkRunType, MarkTestParameters +from tests.functional.utils.test_framework import is_xdist_master + +logger = logging.getLogger(__name__) + + +if enable_pytest_plugins: + pytest_plugins = [ + "tests.functional.fixtures.ovms", + "tests.functional.fixtures.server", + "tests.functional.fixtures.api_type", + "tests.functional.fixtures.params", + ] + + + def pytest_configure(config): + """ + Allow plugins and conftest files to perform initial configuration. + This hook is called for every plugin and initial conftest file after command line options have been parsed. + After that, the hook is called for other conftest files as they are imported. + + NOTE: + This hook is called multiple times: + 1) for master process prior spawning workers + (PYTEST_XDIST_WORKER_COUNT and PYTEST_XDIST_WORKER env variable unset) + 2) for each spawned worker process + + LIMITATIONS: + Internal pytest logging mechanisms are initialized in `pytest_sessionstart` hook. + Please avoid usage of logger in all hooks used in this function. + Please simple print(...) call for printing messages. + """ + hooks.mute_warnings() + MarksRegistry.register(config) + + if is_xdist_master(): + hooks.setup_tmp_repos_dir(config) + validate_port_pool(config) + # master thread pytest_configure call. No xdist worker process spawned yet. + hooks.init_environment(config) + hooks.clear_ovms_capi_artifacts() + hooks.setup_artifacts_dir() + hooks.prepare_ovms_package() + hooks.download_resources_master() + hooks.build_local_resources() + hooks.validate_lock_files() + hooks.list_host_zombie_processes() + # else: # Xdist worker thread + # hooks_utils.download_docker_images() + # hooks_utils.init_ovms_config_retrieved_from_master(config) + + # hooks_utils.setup_nginx() + + # Let know that pytest was successfully configured + config.configured = True + + + @pytest.hookimpl(hookwrapper=True) + def pytest_collection_modifyitems(session, config, items): + """ + Support for running tests with component tags. + Report all test component markers to mongo_reporter. + """ + logger.info("Preparing tests for test session in the following folder: {}".format(session.startdir)) + yield # deselect items in default hook way via keyword ('-k') + + # if config.option.collectonly: + # hooks_utils.log_skip_statistic(items) + + # deselected, all_components, all_requirements = preprocess_collected_items(items) + + # if deselected: + # hooks_utils.deselect_items(items, config, deselected) + + # random.Random(7).shuffle(items) + + + def preprocess_collected_items(items): + deselected = [] + all_components = {} + all_requirements = {} + # try: + # # required_marker_ids, excluded_marker_ids = get_marker_ids_for_test_run() + # for item in items: + # if getattr(item, "callspec", None): + # # Store calculated image for later use. + # ovms_type = item.callspec.params.get(OVMS_TYPE_PARAM_NAME, OvmsType.DOCKER) + # base_os = item.callspec.params.get(BASE_OS_PARAM_NAME, OsType.Ubuntu22) + # if ovms_type == OvmsType.BINARY or ovms_type == OvmsType.CAPI: + # item._image = calculate_ovms_binary_name(base_os=base_os) + # else: + # target_device = item.callspec.params.get( + # TARGET_DEVICE_PARAM_NAME, + # TargetDevice.TARGET_DEVICE_CPU, + # ) + # item._image = calculate_ovms_image_name(target_device=target_device, base_os=base_os) + # # add_dynamic_mark(item) + # test_type = MarkRunType.get_test_type_mark(item) + # # set_timeout_per_test_type(item, test_type) + # # update_parent_markers(item, MarkGeneral.COMPONENTS.mark) + # # update_parent_markers(item, MarkGeneral.REQIDS.mark) + # # update_parent_markers(item, MarkPriority.HIGH.mark) + # # update_parent_markers(item, MarkPriority.MEDIUM.mark) + # # update_parent_markers(item, MarkPriority.LOW.mark) + # # if deselect(item, test_type, required_marker_ids, excluded_marker_ids): + # # deselected.append(item) + # # continue + # # update_markers(item, test_type, all_components, MarkGeneral.COMPONENTS.mark) + # # update_markers(item, test_type, all_requirements, MarkGeneral.REQIDS.mark) + # + # except RuntimeError as e: + # error_msg = str(e) + # logger.exception(error_msg) + # sys.exit(error_msg) + # + # return deselected, all_components, all_requirements + + # def pytest_sessionfinish(session, exitstatus): + # current_test_run = hooks_utils.get_current_test_run() + # logger.info( + # "Finishing test session for test run type: {} in the following folder: {}".format( + # current_test_run, session.startdir + # ) + # ) + # test_status_report_header = f"{SEPARATOR} TEST TYPE STATUS REPORT - BEGIN {SEPARATOR}" + # logger.info(test_status_report_header) + # + # data_to_save = hooks_utils.collect_test_status_data(exitstatus) + # hooks_utils.save_test_data(data_to_save) + # + # test_status_report_footer = f"{SEPARATOR} TEST TYPE STATUS REPORT - END {SEPARATOR}" + # logger.info(test_status_report_footer) + # logger.info("Exit status is: {}".format(str(exitstatus))) + # + # if current_test_run != "": + # hooks_utils.get_test_run_reporter(current_test_run).on_run_end() + # else: + # for reporter in test_run_reporters.values(): # Finish all reporters + # reporter.on_run_end() + + + # def pytest_unconfigure(config): + # if getattr(config, "configured", None) is not True: + # # Check if pytest_configure() was done successfuly, if not: logger would be in invalid state so disable. + # for _logger in logger.manager.loggerDict.values(): + # _logger.disabled = True + # + # try: + # if is_xdist_master(): + # if restler_generate_evidence: + # restler_evidence_name_suffix = f"{release_product_version}_{datetime.now().strftime('%Y%m%d%H%M%S')}" + # shutil.make_archive(f"{restler_evidence_dir}_{restler_evidence_name_suffix}", "zip", + # restler_evidence_dir) + # hooks_utils.remove_ports_reservation(config) + # try: + # shutil.rmtree(config.tmp_repos_dir) + # except PermissionError as e: + # if all([ + # "C:\\" in config.tmp_repos_dir, + # type(e) == PermissionError + # ]): + # # workaround for Windows: https://jira.devtools.intel.com/browse/CVS-161953 + # change_dir_permissions(config.tmp_repos_dir) + # shutil.rmtree(config.tmp_repos_dir) + # hooks.teardown_environment() + # if machine_is_reserved_for_test_session: + # hooks_utils.clear_lockfiles() + # except Exception as e: + # error_msg = str(e) + # print(error_msg) + # sys.exit(error_msg) + +MarksRegistry.MARK_ENUMS.extend([OvmsComponents]) + + +def pytest_sessionstart(session): + logger.info("Starting test session in the following folder: {}".format(session.startdir)) + log_configuration_variables() + session.start_time = time.time() + + +def pytest_generate_tests(metafunc): + if OVMS_TYPE_PARAM_NAME in metafunc.fixturenames: + parametrize_ovms_type(metafunc) + + if USES_MAPPING_PARAM_NAME in metafunc.fixturenames: + parametrize_uses_mapping(metafunc) + + if BASE_OS_PARAM_NAME in metafunc.fixturenames: + parametrize_base_os(metafunc) + + if MarkTestParameters.MODEL_TYPE in metafunc.fixturenames: + parametrize_model_type(metafunc) + elif MarkTestParameters.ALL_MODELS in metafunc.fixturenames: + parametrize_all_models(metafunc) + elif MarkTestParameters.MANY_MODELS in metafunc.fixturenames: + parametrize_many_models(metafunc) + elif MarkTestParameters.ITERATION_INFO in metafunc.fixturenames: + parametrize_iteration_info(metafunc) + elif MarkTestParameters.INPUT_SHAPE in metafunc.fixturenames: + parametrize_input_shape(metafunc) + elif MarkTestParameters.PLUGIN_CONFIG in metafunc.fixturenames: + parametrize_plugin_config(metafunc) + elif TARGET_DEVICE_PARAM_NAME in metafunc.fixturenames: + parametrize_target_device(metafunc) + + if MarkTestParameters.MODEL_AUX_TYPE in metafunc.fixturenames: + parametrize_model_aux_type(metafunc) + + #pytest_runtest_protocol ? move + + # + # def pytest_configure(): + # # Perform initial configuration. + # init_logger() + # + # init_conf_logger = logging.getLogger("init_conf") + # + # container_names = get_containers_with_tests_suffix() + # if container_names: + # init_conf_logger.info("Possible conflicting container names: {} " + # "for given tests_suffix: {}".format(container_names, get_tests_suffix())) + # + # if artifacts_dir: + # os.makedirs(artifacts_dir, exist_ok=True) + # + # + # def pytest_keyboard_interrupt(excinfo): + # clean_hanging_docker_resources() + # Server.stop_all_instances() + # + # + # def pytest_unconfigure(): + # # Perform cleanup. + # cleanup_logger = logging.getLogger("cleanup") + # + # cleanup_logger.info("Cleaning hanging docker resources with suffix: {}".format(get_tests_suffix())) + # clean_hanging_docker_resources() + # + # if test_dir_cleanup: + # cleanup_logger.info("Deleting test directory: {}".format(test_dir)) + # delete_test_directory() + # + # if len(Server.running_instances) > 0: + # logger.warning("Test got unstopped docker instances") + # Server.stop_all_instances() + # + # + # @pytest.hookimpl(hookwrapper=True) + # def pytest_collection_modifyitems(session, config, items): + # yield + # items = reorder_items_by_fixtures_used(session) + # + # + # @pytest.hookimpl(tryfirst=True, hookwrapper=True) + # def pytest_runtest_makereport(item, call): + # outcome = yield + # if call.when == "setup": + # report = outcome.get_result() + # report.test_metadata = {"start": call.start} + # + # + # @pytest.hookimpl(hookwrapper=True) + # def pytest_runtest_call(): + # __tracebackhide__ = True + # try: + # outcome = yield + # finally: + # pass + # exception_catcher("call", outcome) + # + # + # @pytest.hookimpl(hookwrapper=True) + # def pytest_runtest_setup(): + # __tracebackhide__ = True + # try: + # outcome = yield + # finally: + # pass + # exception_catcher("setup", outcome) + # + # + # @pytest.hookimpl(hookwrapper=True) + # def pytest_runtest_teardown(): + # __tracebackhide__ = True + # try: + # outcome = yield + # finally: + # pass + # exception_catcher("teardown", outcome) + # + # + # @pytest.hookimpl(hookwrapper=True) + # def pytest_runtest_teardown(item): + # yield + # # Test finished: remove test item for all fixtures that was used + # for fixture in item._server_fixtures: + # if item in item.session._server_fixtures_to_tests[fixture]: + # item.session._server_fixtures_to_tests[fixture].remove(item) + # if len(item.session._server_fixtures_to_tests[fixture]) == 0: + # # No other tests will use this docker instance so we can close it. + # Server.stop_by_fixture_name(fixture) + # + # + # def exception_catcher(when: str, outcome): + # if isinstance(outcome.excinfo, tuple): + # if len(outcome.excinfo) > 1 and isinstance(outcome.excinfo[1], OutcomeException): + # return + # exception_logger = logging.getLogger("exception_logger") + # exception_info = ExceptionInfo.from_exc_info(outcome.excinfo) + # exception_info.traceback = exception_info.traceback.filter(filter_traceback) + # exc_repr = exception_info.getrepr(style="short", chain=False)\ + # if exception_info.traceback\ + # else exception_info.exconly() + # exception_logger.error('Unhandled Exception during {}: \n{}' + # .format(when.capitalize(), str(exc_repr))) + # + # + # @pytest.hookimpl(hookwrapper=True, tryfirst=True) + # def pytest_runtest_logstart(nodeid, location): + # if artifacts_dir: + # test_name = get_path_friendly_test_name(location) + # log_path = os.path.join(artifacts_dir, f"{test_name}.log") + # _root_logger = logging.getLogger(None) + # _root_logger._test_log_handler = FileHandler(log_path) + # formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s") + # _root_logger._test_log_handler.setFormatter(formatter) + # _root_logger.addHandler(_root_logger._test_log_handler) + # yield + # + # + # @pytest.hookimpl(hookwrapper=True, tryfirst=True) + # def pytest_runtest_logfinish(nodeid, location): + # if artifacts_dir: + # _root_logger = logging.getLogger(None) + # _root_logger.removeHandler(_root_logger._test_log_handler) + # yield + diff --git a/tests/functional/constants/os_version.py b/tests/functional/constants/os_version.py new file mode 100644 index 0000000000..bdbbf4e796 --- /dev/null +++ b/tests/functional/constants/os_version.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.constants.os_type import OsType + +REDHAT_MINIMAL_BASE_IMAGE = "registry.access.redhat.com/ubi9/ubi-minimal:9.7" +REDHAT_COMMON_BASE_IMAGE = "registry.access.redhat.com/ubi9/ubi:9.7" +UBUNTU_22_BASE_IMAGE = "ubuntu:22.04" +UBUNTU_24_BASE_IMAGE = "ubuntu:24.04" + +os_type_to_base_image = { + OsType.Redhat: REDHAT_COMMON_BASE_IMAGE, + OsType.Ubuntu22: UBUNTU_22_BASE_IMAGE, + OsType.Ubuntu24: UBUNTU_24_BASE_IMAGE, +} + +os_type_to_base_image_binary_docker = { + OsType.Redhat: REDHAT_COMMON_BASE_IMAGE, + OsType.Ubuntu22: UBUNTU_22_BASE_IMAGE, + OsType.Ubuntu24: UBUNTU_24_BASE_IMAGE +} + +OPENVINO_UBUNTU_20_DEV_IMAGE = "openvino/ubuntu20_dev:2024.6.0" +OPENVINO_MODEL_SERVER_LATEST = "openvino/model_server:latest" +OPENVINO_MODEL_SERVER_LATEST_GPU = "openvino/model_server:latest-gpu" +OPENVINO_MODEL_SERVER_LATEST_PY = "openvino/model_server:latest-py" +OPENVINO_MODEL_SERVER_WEEKLY = "openvino/model_server:weekly" +NO_DOC_UPDATE_IMAGES = [OPENVINO_MODEL_SERVER_LATEST, OPENVINO_MODEL_SERVER_LATEST_GPU, + OPENVINO_MODEL_SERVER_LATEST_PY, OPENVINO_MODEL_SERVER_WEEKLY] diff --git a/tests/functional/constants/ovms_images.py b/tests/functional/constants/ovms_images.py index 1e268965b4..dcfc06367e 100644 --- a/tests/functional/constants/ovms_images.py +++ b/tests/functional/constants/ovms_images.py @@ -19,13 +19,15 @@ from tests.functional.utils.environment_info import EnvironmentInfo from tests.functional.constants.os_type import OsType, UBUNTU from tests.functional.config import ( + base_os, docker_registry, + force_use_ovms_image, is_nginx_mtls, ovms_cpp_docker_image, ovms_image, ovms_image_tag, ovms_test_image_name, - force_use_ovms_image, + target_devices, ) from tests.functional.constants.target_device import TargetDevice from tests.functional.constants.ovms import CurrentOvmsType @@ -172,3 +174,11 @@ def calculate_ovms_capi_image_name(ovms_image_name): def calculate_ovms_test_image_name(ovms_image_name): test_image_name = ovms_test_image_name if ovms_test_image_name is not None else f"{ovms_image_name}-test" return test_image_name + + +def get_ovms_calculated_images(): + ovms_images = set() + for _os in base_os: + for _target_device in target_devices: + ovms_images.add((calculate_ovms_image_name(_target_device, _os), _os)) + return ovms_images diff --git a/tests/functional/constants/paths.py b/tests/functional/constants/paths.py index 6a0636d3a5..9824023a9f 100644 --- a/tests/functional/constants/paths.py +++ b/tests/functional/constants/paths.py @@ -63,6 +63,17 @@ class Paths: def CAPI_WRAPPER_PACKAGE_CONTENT_PATH(base_os): return os.path.join(config.c_api_wrapper_dir, base_os, "ovms") + COMMON_GIT_CLONE_LOCK_FILE = os.path.join(config.ovms_file_locks_dir, "common_git_clone.lock") + + COMMON_BUILD_LOCK_FILE = os.path.join(config.ovms_file_locks_dir, "common_build.lock") + + # Use single shared lock file until ensure that those builds can be done concurrently. + DOCKER_BUILD_LOCK_FILE = COMMON_BUILD_LOCK_FILE + CUSTOM_NODE_BUILD_LOCK_FILE = COMMON_BUILD_LOCK_FILE + CPU_EXTENSION_BUILD_LOCK_FILE = COMMON_BUILD_LOCK_FILE + + COMMON_DOWNLOAD_LOCK_FILE = os.path.join(config.ovms_file_locks_dir, "common_download.lock") + @staticmethod def get_target_device_lock_file(target_device, i): if isinstance(target_device, str): diff --git a/tests/functional/utils/docker.py b/tests/functional/utils/docker.py index 51f86e70dd..35d5a472b0 100644 --- a/tests/functional/utils/docker.py +++ b/tests/functional/utils/docker.py @@ -33,6 +33,8 @@ from tests.functional.config import docker_client_timeout from tests.functional.constants.core import CONTAINER_STATUS_RUNNING +DOCKER_CONTAINER_TMP_PATH = "/tmp" + logger = get_logger(__name__) diff --git a/tests/functional/utils/download.py b/tests/functional/utils/download.py new file mode 100644 index 0000000000..17f40d39ec --- /dev/null +++ b/tests/functional/utils/download.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pathlib import Path + +from tests.functional.utils.core import SelfDeletingFileLock +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process + +logger = get_logger(__name__) + + +def wget_item(dst, cmd): + Path(dst).parent.mkdir(parents=True, exist_ok=True) + proc = Process() + proc.set_log_silence() + proc.policy["log-check-output"]["stderr"] = False + + with SelfDeletingFileLock(f"{dst}.lock", self_delete=True) as fl: + proc.run_and_check(cmd) + + +def wget_file(url, dst): + logger.info(f"Downloading file via wget\n{url} => {dst}") + cmd = f"wget {url} -O {dst}" + wget_item(dst, cmd) + + +def curl_file(url, dst, user, token): + logger.info(f"Downloading file via curl\n{url} => {dst}") + cmd = f'curl --insecure -L --user {user}:{token} "{url}" -o {dst}' + proc = Process() + proc.set_log_silence() + proc.policy["log-check-output"]["stderr"] = False + with SelfDeletingFileLock(f"{dst}.lock", self_delete=True) as fl: + proc.run_and_check(cmd) + + +def wget_folder(url, dst, depth=2, reject=".html,.tmp", extra_options=None): + logger.info(f"Downloading folder via wget\n{url} => {dst}") + options = f"-r --directory-prefix={dst} --no-parent --no-host-directories --cut-dirs={depth} --reject={reject}" + if extra_options is not None: + options += extra_options + cmd = f"wget {options} {url}" + wget_item(dst, cmd) diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index 409b24570f..376fbecd9e 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -16,41 +16,63 @@ import os import shutil +import sys import warnings from collections import defaultdict from docker import errors as docker_errors from pathlib import Path - from tests.functional import config from ovms.constants.models_library import ModelsLib +from tests.functional.utils.download import wget_file from tests.functional.utils.reservation_manager.args import parse_args from tests.functional.utils.reservation_manager.manager import Manager as ReservationManager from tests.functional.config import ( + build_test_image, c_api_wrapper_dir, cleanup_env_on_startup, global_tmp_dir_default, + http_proxy, + https_proxy, + no_proxy, ovms_c_repo_path, machine_is_reserved_for_test_session, tmp_dir, ) -from tests.functional.constants.os_type import get_host_os, OsType +from tests.functional.constants.os_type import get_host_os, OsType, UBUNTU +from tests.functional.constants.os_version import os_type_to_base_image_binary_docker from tests.functional.constants.ovms import ( BASE_OS_PARAM_NAME, OVMS_TYPE_PARAM_NAME, TARGET_DEVICE_PARAM_NAME, USES_MAPPING_PARAM_NAME, ) -from tests.functional.constants.ovms_images import calculate_ovms_image_name +from tests.functional.constants.ovms_images import ( + calculate_ovms_binary_image_name, + calculate_ovms_capi_image_name, + calculate_ovms_test_image_name, + calculate_ovms_image_name, + get_ovms_calculated_images, + GPU_INSTALL_DRIVER_VERSION, + GPU_INSTALL_SCRIPTS, +) +from tests.functional.constants.ovms_type import ( + OvmsType, + OVMS_BINARY_DEPENDENCIES, + OVMS_BINARY_PACKAGE_EXTENSIONS, + OVMS_BINARY_PACKAGE_NAME, + OVMS_CAPI_DEPENDENCIES, + OVMS_CAPI_TOOLS_DEPENDENCIES, +) from tests.functional.constants.paths import Paths -from tests.functional.constants.target_device import TargetDevice +from tests.functional.constants.target_device import MAX_WORKERS_PER_TARGET_DEVICE, TargetDevice from tests.functional.object_model.ovms_info import OvmsInfo from tests.functional.utils.core import TmpDir -from tests.functional.utils.docker import DockerClient, DockerContainer +from tests.functional.utils.docker import DockerClient, DockerContainer, DOCKER_CONTAINER_TMP_PATH from tests.functional.utils.environment_info import EnvironmentInfo from tests.functional.utils.logger import get_logger from tests.functional.utils.marks import MarkTestParameters -from tests.functional.utils.process import Process +from tests.functional.utils.process import PID_STATE_ZOMBIE, Process, get_pid_name, get_pid_status from tests.functional.utils.test_framework import get_test_object_prefix logger = get_logger(__name__) @@ -169,6 +191,188 @@ def setup_artifacts_dir(): file.unlink() +def setup_capi_wrapper(package_content): + if OvmsType.CAPI not in config.ovms_types: + return + for _src, _dst in [ + (Paths.OVMS_TEST_CAPI_WRAPPER_PYX, Path(package_content, "include")), + (Paths.OVMS_TEST_CAPI_WRAPPER_MAKEFILE, package_content), + (Paths.OVMS_TEST_CAPI_WRAPPER_SETUP, package_content), + (Paths.OVMS_TEST_CAPI_AUTOPXD_PY, Path(package_content, "include")), + ]: + shutil.copy(_src, _dst) + + proc = Process() + proc.disable_check_stderr() + + # Example: + # >>> sys.executable + # '/usr/local/ovms-test/.venv/bin/python3' + # >>> venv_path + # '/usr/local/ovms-test/.venv' + venv_path = str(Path(*Path(sys.executable).parts[:-2])) + _stdout = proc.run_and_check(f"PYVENV={venv_path} make", cwd=package_content) + + +def download_binary_package(binary_package_src_file_path, binary_package_dst_file_path): + wget_file(binary_package_src_file_path, binary_package_dst_file_path) + + +def get_binary_artifacts( + binary_package_src_file_path, binary_package_dst_path, ovms_binary_name=OVMS_BINARY_PACKAGE_NAME +): + proc = Process() + proc.disable_check_stderr() + print(f"Preparing OVMS package: {binary_package_src_file_path}") + used_extensions = [extension for extension in OVMS_BINARY_PACKAGE_EXTENSIONS + if binary_package_src_file_path.endswith(extension)] + if not used_extensions: + raise NotImplementedError( + f"OVMS binary supported only with .tar.gz or .zip formats. " + f"Current package name: {binary_package_src_file_path}" + ) + ovms_binary_full_name = f"{ovms_binary_name}{used_extensions[0]}" + + if binary_package_src_file_path.startswith("http"): + binary_package_dst_file_path = os.path.join(binary_package_dst_path, ovms_binary_full_name) + download_binary_package(binary_package_src_file_path, binary_package_dst_file_path) + else: + ovms_binary_src_path = os.path.realpath(os.path.expanduser(binary_package_src_file_path)) + if not os.path.exists(binary_package_dst_path): + os.makedirs(binary_package_dst_path) + shutil.copy(ovms_binary_src_path, os.path.join(binary_package_dst_path, ovms_binary_full_name)) + proc.run_and_check(f"tar -xf {ovms_binary_full_name}", cwd=binary_package_dst_path) + setupvars_script_dst = os.path.join(binary_package_dst_path, "ovms", "setupvars.bat") + if not os.path.exists(setupvars_script_dst): + shutil.copy2(config.setupvars_script_path, setupvars_script_dst) + + +def run_docker_build_ovms_image(cmd, ovms_image_name, cwd, timeout=None): + print(f"Building {ovms_image_name} image using cmd: {cmd}") + proc = Process() + proc.disable_check_stderr() + code, stdout, stderr = proc.run_and_check_return_all(cmd, cwd=cwd, timeout=timeout) + assert (f"naming to {ovms_image_name}" in stderr) or ( + f"Successfully tagged {ovms_image_name}" in stdout + ), f"Image was not built successfully; stderr: {stderr}" + print(f"Ovms-test image {ovms_image_name} successfully created") + + +def get_ovms_capi_docker_build_cmd(ovms_image, base_os, dockerfile, ovms_binary_image_name): + base_image = config.base_image if config.base_image else os_type_to_base_image_binary_docker[base_os] + ovms_test_image = calculate_ovms_test_image_name(ovms_image) if build_test_image else base_image + target_device = TargetDevice.GPU if TargetDevice.GPU.lower() in ovms_image else TargetDevice.CPU + cmd = ( + f"docker build -f {dockerfile} -t {ovms_binary_image_name} . " + f"--build-arg BASE_IMAGE={base_image} " + f"--build-arg OVMS_IMAGE={ovms_image} " + f"--build-arg OVMS_TEST_IMAGE={ovms_test_image} " + f"--build-arg OVMS_DEPENDENCIES='{OVMS_CAPI_DEPENDENCIES[base_os]}' " + f"--build-arg TOOLS_DEPENDENCIES='{OVMS_CAPI_TOOLS_DEPENDENCIES[target_device][base_os]}' " + f"--build-arg INSTALL_DRIVER_VERSION='{GPU_INSTALL_DRIVER_VERSION[base_os]}' " + f"--build-arg http_proxy={http_proxy} " + f"--build-arg https_proxy={https_proxy} " + f"--build-arg no_proxy={no_proxy} " + ) + return cmd + + +def get_ovms_binary_docker_build_cmd(ovms_image, base_os, dockerfile, ovms_binary_image_name): + base_image = config.base_image if config.base_image else os_type_to_base_image_binary_docker[base_os] + ovms_test_image = calculate_ovms_test_image_name(ovms_image) if build_test_image else base_image + cpu_extensions_path = Paths.ROOT_PATH_CPU_EXTENSIONS if build_test_image else DOCKER_CONTAINER_TMP_PATH + custom_loader_path = Paths.CUSTOM_LOADER_LIBRARIES_PATH_INTERNAL if build_test_image else DOCKER_CONTAINER_TMP_PATH + custom_nodes_path = Paths.CUSTOM_NODE_LIBRARIES_PATH_INTERNAL if build_test_image else DOCKER_CONTAINER_TMP_PATH + cmd = ( + f"docker build -f {dockerfile} -t {ovms_binary_image_name} . " + f"--build-arg BASE_IMAGE={base_image} " + f"--build-arg OVMS_IMAGE={ovms_image} " + f"--build-arg OVMS_TEST_IMAGE={ovms_test_image} " + f"--build-arg OVMS_DEPENDENCIES='{OVMS_BINARY_DEPENDENCIES[base_os]}' " + f"--build-arg CPU_EXTENSIONS_PATH={cpu_extensions_path} " + f"--build-arg CUSTOM_LOADER_PATH={custom_loader_path} " + f"--build-arg CUSTOM_NODES_PATH={custom_nodes_path} " + f"--build-arg http_proxy={http_proxy} " + f"--build-arg https_proxy={https_proxy} " + f"--build-arg no_proxy={no_proxy} " + ) + return cmd + + +def build_ovms_binary_image(): + ovms_c_artifacts = {_base_os: config.ovms_c_release_artifacts_path[index] for index, _base_os in enumerate(config.base_os)} + + for ovms_image, base_os in get_ovms_calculated_images(): + ovms_binary_dst_path = os.path.join(tmp_dir, "ovms_binary", base_os) + get_binary_artifacts(ovms_c_artifacts[base_os], ovms_binary_dst_path) + + dockerfile = f"Dockerfile.{UBUNTU if UBUNTU in base_os else base_os}" + shutil.copy( + os.path.join(os.path.dirname(__file__), "ovms_binary_image", dockerfile), + ovms_binary_dst_path, + ) + + ovms_binary_image_name = calculate_ovms_binary_image_name(ovms_image) + cmd = get_ovms_binary_docker_build_cmd(ovms_image, base_os, dockerfile, ovms_binary_image_name) + run_docker_build_ovms_image(cmd, ovms_binary_image_name, cwd=ovms_binary_dst_path, timeout=None) + + +def build_ovms_capi_image(): + ovms_c_artifacts = {_base_os: config.ovms_c_release_artifacts_path[index] for index, _base_os in enumerate(config.base_os)} + + for ovms_image, base_os in get_ovms_calculated_images(): + ovms_capi_dst_path = os.path.join(tmp_dir, "ovms_capi", base_os) + get_binary_artifacts(ovms_c_artifacts[base_os], ovms_capi_dst_path) + if TargetDevice.GPU.lower() in ovms_image: + for gpu_install_script in GPU_INSTALL_SCRIPTS[base_os]: + shutil.copy(os.path.join(ovms_c_repo_path, gpu_install_script), ovms_capi_dst_path) + else: + for gpu_install_script in GPU_INSTALL_SCRIPTS[base_os]: + with open(os.path.join(ovms_capi_dst_path, gpu_install_script), "a"): + pass + + dockerfile = f"Dockerfile.{UBUNTU if UBUNTU in base_os else base_os}" + shutil.copy( + os.path.join(os.path.dirname(__file__), "ovms_capi_image", dockerfile), + ovms_capi_dst_path, + ) + + ovms_capi_image_name = calculate_ovms_capi_image_name(ovms_image) + cmd = get_ovms_capi_docker_build_cmd(ovms_image, base_os, dockerfile, ovms_capi_image_name) + run_docker_build_ovms_image(cmd, ovms_capi_image_name, cwd=ovms_capi_dst_path, timeout=None) + + +def prepare_ovms_package(): + if all([ + all([OvmsType.CAPI not in ovms_type for ovms_type in config.ovms_types]), + all([OvmsType.BINARY not in ovms_type for ovms_type in config.ovms_types]), + ]): + return + + if any([OvmsType.CAPI in config.ovms_types, OvmsType.BINARY in config.ovms_types]): + assert ( + len(config.base_os) == 1 and get_host_os() == config.base_os[0] + ), f"Mismatch between config base os: {config.base_os}; host os: {get_host_os()}" + + for base_os in config.base_os: + ovms_binary_dst_path = os.path.join(c_api_wrapper_dir, base_os) + get_binary_artifacts(config.ovms_c_release_artifacts_path[0], ovms_binary_dst_path) + + package_content = Path(Paths.CAPI_WRAPPER_PACKAGE_CONTENT_PATH(base_os)) + setup_capi_wrapper(package_content) + + +def download_resources_master(): + print("Download required resources") + + +def build_local_resources(): + if OvmsType.BINARY_DOCKER in config.ovms_types: + build_ovms_binary_image() + if OvmsType.CAPI_DOCKER in config.ovms_types: + build_ovms_capi_image() + + def setup_tmp_repos_dir(config): config.tmp_repos_dir = TmpDir() @@ -314,6 +518,35 @@ def parametrize_target_device(metafunc): metafunc.parametrize(TARGET_DEVICE_PARAM_NAME, config.target_devices, ids=ids) +def validate_lock_files(): + """Ensure that target_device locks files exists""" + if not config.machine_is_reserved_for_test_session: + return # Cannot validate locks validity since other testing session could acquire device lock + + locks = [value for key, value in vars(Paths).items() if "LOCK_FILE" in key] + for target_device in config.target_devices: + n = MAX_WORKERS_PER_TARGET_DEVICE[target_device] + locks += [Paths.get_target_device_lock_file(target_device, i) for i in range(n)] + for lock_path in [Path(x) for x in locks]: + if lock_path.exists(): + logger.warning(f"Hanging lock file discovered:\n{lock_path.name}") + logger.warning(f"Deleting lock file:\n{lock_path.name}") + lock_path.unlink() + + +def list_host_zombie_processes(): + zombie_pids = [] + if OsType.Windows in config.base_os: + return zombie_pids + all_pids = [x.name for x in Path("/proc").iterdir() if str(x.name).isnumeric()] + zombie_pids = [x for x in all_pids if get_pid_status(x) == PID_STATE_ZOMBIE] + if len(zombie_pids) > 0: + logger.warning(f"Found {len(zombie_pids)} zombie processes.") + for zombie in zombie_pids: + logger.warning(f"Zombie:\t{get_pid_name(zombie)}") + return zombie_pids + + def parametrize_ovms_type(metafunc): metafunc.parametrize(OVMS_TYPE_PARAM_NAME, config.ovms_types) diff --git a/tests/functional/utils/ovms_binary_image/Dockerfile.redhat b/tests/functional/utils/ovms_binary_image/Dockerfile.redhat new file mode 100644 index 0000000000..31e2da11d3 --- /dev/null +++ b/tests/functional/utils/ovms_binary_image/Dockerfile.redhat @@ -0,0 +1,47 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.7 +ARG OVMS_TEST_IMAGE=openvino-model-server:latest-test +ARG OVMS_IMAGE=registry.connect.redhat.com/intel/openvino-model-server:latest + +FROM $OVMS_IMAGE as ovms_image +FROM $OVMS_TEST_IMAGE as ovms_test_image +FROM $BASE_IMAGE as base_image + +# libtbb is now available in /ovms/lib +#ARG OVMS_DEPENDENCIES="https://vault.centos.org/centos/8/AppStream/x86_64/os/Packages/tbb-2018.2-9.el8.x86_64.rpm" +#RUN dnf install -y pkg-config && rpm -ivh ${OVMS_DEPENDENCIES} + +WORKDIR / +COPY ovms/ ovms/ + +ENV LD_LIBRARY_PATH=/ovms/lib + +RUN /ovms/bin/ovms --version + +# Copy all dependent libraries for enabling GPU based images (be cautious) +COPY --from=ovms_image /etc /etc +COPY --from=ovms_image /usr/lib64/ /usr/lib64/ +COPY --from=ovms_image /usr/local/ /usr/local/ + +ARG CPU_EXTENSIONS_PATH=/cpu_extensions +ARG CUSTOM_NODES_PATH=/custom_nodes + +COPY --from=ovms_test_image ${CPU_EXTENSIONS_PATH} ${CPU_EXTENSIONS_PATH} +COPY --from=ovms_test_image ${CUSTOM_NODES_PATH} ${CUSTOM_NODES_PATH} + +ENTRYPOINT ["/ovms/bin/ovms"] diff --git a/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu b/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu new file mode 100644 index 0000000000..2aba99d708 --- /dev/null +++ b/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu @@ -0,0 +1,46 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=ubuntu:24.04 +ARG OVMS_TEST_IMAGE=openvino/model_server:latest-test +ARG OVMS_IMAGE=openvino/model_server:latest + +FROM $OVMS_IMAGE as ovms_image +FROM $OVMS_TEST_IMAGE as ovms_test_image +FROM $BASE_IMAGE as base_image + +ARG OVMS_DEPENDENCIES="libcurl4-openssl-dev libpugixml1v5 libtbb2 libxml12" +RUN apt-get update && apt-get install -y --no-install-recommends ${OVMS_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* + +WORKDIR / +COPY ovms/ ovms/ + +ENV LD_LIBRARY_PATH=/ovms/lib + +RUN /ovms/bin/ovms --version + +# Copy all dependent libraries for enabling GPU based images (be cautious) +COPY --from=ovms_image /etc /etc +COPY --from=ovms_image /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu +COPY --from=ovms_image /usr/local/ /usr/local/ + +ARG CPU_EXTENSIONS_PATH=/cpu_extensions +ARG CUSTOM_NODES_PATH=/custom_nodes + +COPY --from=ovms_test_image ${CPU_EXTENSIONS_PATH} ${CPU_EXTENSIONS_PATH} +COPY --from=ovms_test_image ${CUSTOM_NODES_PATH} ${CUSTOM_NODES_PATH} + +ENTRYPOINT ["/ovms/bin/ovms"] diff --git a/tests/functional/utils/ovms_capi_image/Dockerfile.redhat b/tests/functional/utils/ovms_capi_image/Dockerfile.redhat new file mode 100644 index 0000000000..393121890f --- /dev/null +++ b/tests/functional/utils/ovms_capi_image/Dockerfile.redhat @@ -0,0 +1,47 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.7 +ARG OVMS_TEST_IMAGE=openvino-model-server:latest-test +ARG OVMS_IMAGE=registry.connect.redhat.com/intel/openvino-model-server:latest + +FROM $OVMS_IMAGE as ovms_image +FROM $OVMS_TEST_IMAGE as ovms_test_image +FROM $BASE_IMAGE as base_image + +# libtbb is now available in /ovms/lib +#ARG OVMS_DEPENDENCIES="https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/tbb-2020.3-8.el9.x86_64.rpm" +#RUN dnf install -y pkg-config && rpm -ivh ${OVMS_DEPENDENCIES} + +WORKDIR / +COPY ovms/ ovms/ + +ENV LD_LIBRARY_PATH=/ovms/lib + +RUN /ovms/bin/ovms --version +RUN dnf install -y https://rpmfind.net/linux/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libnl3-3.11.0-1.el9.x86_64.rpm && dnf clean all +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && dnf clean all && \ + yum update -d6 -y && yum install -d6 -y gcc-c++ +RUN dnf install -y pkg-config && rpm -ivh https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/opencl-headers-3.0-6.20201007gitd65bcc5.el9.noarch.rpm && dnf clean all + +# Enable GPU +ARG INSTALL_DRIVER_VERSION="24.52.32224" +ARG DNF_TOOL="dnf" +COPY install_redhat_gpu_drivers.sh /tmp/install_redhat_gpu_drivers.sh +RUN chmod 775 /tmp/install_redhat_gpu_drivers.sh && /tmp/install_redhat_gpu_drivers.sh + +# use for debug with GPU target device +#RUN yum update -d6 -y && yum install -d6 -y clinfo diff --git a/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu b/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu new file mode 100644 index 0000000000..1fb7d4b5c7 --- /dev/null +++ b/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu @@ -0,0 +1,44 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=ubuntu:24.04 +ARG OVMS_TEST_IMAGE=openvino/model_server:latest-test +ARG OVMS_IMAGE=openvino/model_server:latest + +FROM $OVMS_IMAGE as ovms_image +FROM $OVMS_TEST_IMAGE as ovms_test_image +FROM $BASE_IMAGE as base_image + +ARG OVMS_DEPENDENCIES="libcurl4-openssl-dev libpugixml1v5 libtbb2 libxml12" +RUN apt-get update && apt-get install -y --no-install-recommends ${OVMS_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* + +WORKDIR / +COPY ovms/ ovms/ + +ENV LD_LIBRARY_PATH=/ovms/lib + +RUN /ovms/bin/ovms --version + +ARG TOOLS_DEPENDENCIES="build-essential" +RUN apt-get update && apt-get install -y --no-install-recommends ${TOOLS_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* + +# Enable GPU +ARG INSTALL_DRIVER_VERSION="25.35.35096" +COPY install_ubuntu_gpu_drivers.sh /tmp/install_ubuntu_gpu_drivers.sh +RUN chmod 775 /tmp/install_ubuntu_gpu_drivers.sh && /tmp/install_ubuntu_gpu_drivers.sh + +#COPY install_va.sh /tmp/install_va.sh +#RUN chmod 775 /tmp/install_va.sh && /tmp/install_va.sh diff --git a/tests/functional/utils/ovms_testing_image/Dockerfile.redhat b/tests/functional/utils/ovms_testing_image/Dockerfile.redhat index 6d89e5483c..c38e61f277 100644 --- a/tests/functional/utils/ovms_testing_image/Dockerfile.redhat +++ b/tests/functional/utils/ovms_testing_image/Dockerfile.redhat @@ -14,7 +14,7 @@ # limitations under the License. # -ARG BASE_IMAGE=openvino-model-server:latest +ARG BASE_IMAGE=registry.connect.redhat.com/intel/openvino-model-server:latest FROM $BASE_IMAGE as base_image ARG ROOT_PATH_CPU_EXTENSIONS=/cpu_extensions From 1229591c199d4ceffada0b2387fa33592111fa0c Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Tue, 16 Jun 2026 16:00:05 +0200 Subject: [PATCH 02/13] 16/06/2026 --- tests/functional/config.py | 10 ++- tests/functional/utils/hooks.py | 117 +++++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/tests/functional/config.py b/tests/functional/config.py index 95abe9772c..0874817cd4 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -147,8 +147,11 @@ def get_uses_mapping(): path_to_mount_cache = os.path.join(test_dir_cache, "saved_models") -"""TT_MINIO_IMAGE_NAME - Docker image for Minio""" -minio_image = os.environ.get("TT_MINIO_IMAGE_NAME", "minio/minio:latest") +""" TT_MINIO_IMAGE_NAME - Docker image for Minio""" +minio_image = os.environ.get( + "TT_MINIO_IMAGE_NAME", + f"{docker_registry}/minio/minio:latest" if docker_registry is not None else "minio/minio:latest", +) """ TT_TARGET_DEVICE - list of devices separated by a comma "CPU,GPU,NPU" """ target_devices = get_target_devices() @@ -167,6 +170,9 @@ def get_uses_mapping(): """ TT_IS_NGINX_MTLS - Specify if given image is OVSA nginx mtls image. """ is_nginx_mtls = get_bool("TT_IS_NGINX_MTLS", False) +""" TT_FORCE_GENERATE_NEW_SSL_CERTIFICATES """ +force_generate_new_ssl_certs = get_bool("TT_FORCE_GENERATE_NEW_SSL_CERTIFICATES", True) + """ TT_SKIP_TEST_IF_IS_NGINX_MTLS """ skip_nginx_test = get_bool("TT_SKIP_TEST_IF_IS_NGINX_MTLS", True) skip_nginx_test = skip_nginx_test and is_nginx_mtls diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index 376fbecd9e..44e7d8cc06 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -14,6 +14,7 @@ # limitations under the License. # +import itertools import os import shutil import sys @@ -31,11 +32,14 @@ build_test_image, c_api_wrapper_dir, cleanup_env_on_startup, + force_generate_new_ssl_certs, global_tmp_dir_default, http_proxy, https_proxy, + is_nginx_mtls, no_proxy, ovms_c_repo_path, + ovms_file_locks_dir, machine_is_reserved_for_test_session, tmp_dir, ) @@ -43,8 +47,10 @@ from tests.functional.constants.os_version import os_type_to_base_image_binary_docker from tests.functional.constants.ovms import ( BASE_OS_PARAM_NAME, + CURRENT_TARGET_DEVICE_DICT_ARGUMENT, OVMS_TYPE_PARAM_NAME, TARGET_DEVICE_PARAM_NAME, + TMP_REPOS_DIR_ARGUMENT, USES_MAPPING_PARAM_NAME, ) from tests.functional.constants.ovms_images import ( @@ -73,7 +79,8 @@ from tests.functional.utils.logger import get_logger from tests.functional.utils.marks import MarkTestParameters from tests.functional.utils.process import PID_STATE_ZOMBIE, Process, get_pid_name, get_pid_status -from tests.functional.utils.test_framework import get_test_object_prefix +from tests.functional.utils.test_framework import change_dir_permissions, get_test_object_prefix, is_xdist_master +from tests.functional.object_model.ovsa import OvsaCerts logger = get_logger(__name__) @@ -145,6 +152,15 @@ def cleanup_docker_images(): logger.info(f"Removed docker image: {image.id}") +def cleanup_tmp_repos_dir(): + try: + shutil.rmtree(config.tmp_repos_dir) + except PermissionError as e: + if get_host_os() == OsType.Windows and type(e) == PermissionError: + change_dir_permissions(config.tmp_repos_dir) + shutil.rmtree(config.tmp_repos_dir) + + def teardown_environment(): if get_host_os() == OsType.Windows: if config.teardown_ovms_processes: @@ -362,9 +378,45 @@ def prepare_ovms_package(): setup_capi_wrapper(package_content) +def get_docker_images(images_to_download): + if OsType.Windows in config.base_os: + return + docker_ovms_types = [ + OvmsType.DOCKER, OvmsType.DOCKER_CMD_LINE, OvmsType.BINARY_DOCKER, OvmsType.CAPI_DOCKER + ] + if not any(_ovms_type in docker_ovms_types for _ovms_type in config.ovms_types): + return + + images_to_download.add(config.minio_image) + for target_device, base_os in itertools.product(config.target_devices, config.base_os): + ovms_image = calculate_ovms_image_name(target_device, base_os) + if config.ovms_image_local: + OvmsInfo.get_local_image(ovms_image) + else: + images_to_download.add(ovms_image) + return images_to_download + + +def download_docker_images(): + images_to_download = set() + images_to_download = get_docker_images(images_to_download) + + for image in images_to_download: + if image: + OvmsInfo.pull_latest_image(image) + + def download_resources_master(): print("Download required resources") + download_docker_images() + + +def init_ovms_config_retrieved_from_master(pytest_config): + config.tmp_repos_dir = pytest_config.workerinput[TMP_REPOS_DIR_ARGUMENT] + global CURRENT_TARGET_DEVICE_DICT + CURRENT_TARGET_DEVICE_DICT = pytest_config.workerinput[CURRENT_TARGET_DEVICE_DICT_ARGUMENT] + def build_local_resources(): if OvmsType.BINARY_DOCKER in config.ovms_types: @@ -647,3 +699,66 @@ def mute_warnings(): warnings.filterwarnings( action="ignore", message="`np.bool8` is a deprecated alias for `np.bool_`", category=DeprecationWarning ) + + +def setup_nginx(): + if not is_nginx_mtls: + return + print("Setup ngnix certificates") + if is_xdist_master(): + OvsaCerts.generate_ovsa_certs(skip_if_valid=not force_generate_new_ssl_certs) + OvsaCerts.init_ovsa_certs() + + +def remove_ports_reservation(_config): + reservation_manager = getattr(_config, "reservation_manager", None) + if reservation_manager is None: + return + logger.info("Removing self reserved ports") + reservation_manager.independent.remove() + + if machine_is_reserved_for_test_session: + # If machine is reserved, no other test session should active + # So clean dangling reservations (if any occurs during previous fatal errors). + # If machine never be reserved exclusively (ie. builder0x) + # you can clean dangling reservation manually by deleting files: + # `/tmp/reservation_manager-*-*-independent` + # (after ensuring no test session is active) + reservation_manager.independent.cleanup() + + +def clear_lockfiles(): + if not Path(ovms_file_locks_dir).exists(): + return + for file in Path(ovms_file_locks_dir).iterdir(): + print(f"Delete hanging lock: {str(file)}") # logger could be unavailable by now + file.unlink() + + +def parametrize_tests(metafunc): + if OVMS_TYPE_PARAM_NAME in metafunc.fixturenames: + parametrize_ovms_type(metafunc) + + if USES_MAPPING_PARAM_NAME in metafunc.fixturenames: + parametrize_uses_mapping(metafunc) + + if BASE_OS_PARAM_NAME in metafunc.fixturenames: + parametrize_base_os(metafunc) + + if MarkTestParameters.MODEL_TYPE in metafunc.fixturenames: + parametrize_model_type(metafunc) + elif MarkTestParameters.ALL_MODELS in metafunc.fixturenames: + parametrize_all_models(metafunc) + elif MarkTestParameters.MANY_MODELS in metafunc.fixturenames: + parametrize_many_models(metafunc) + elif MarkTestParameters.ITERATION_INFO in metafunc.fixturenames: + parametrize_iteration_info(metafunc) + elif MarkTestParameters.INPUT_SHAPE in metafunc.fixturenames: + parametrize_input_shape(metafunc) + elif MarkTestParameters.PLUGIN_CONFIG in metafunc.fixturenames: + parametrize_plugin_config(metafunc) + elif TARGET_DEVICE_PARAM_NAME in metafunc.fixturenames: + parametrize_target_device(metafunc) + + if MarkTestParameters.MODEL_AUX_TYPE in metafunc.fixturenames: + parametrize_model_aux_type(metafunc) From eb437e17da3e732a7d9bde289cc78d94e2d8373d Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Wed, 17 Jun 2026 19:15:47 +0200 Subject: [PATCH 03/13] 17/06/2026 --- tests/functional/config.py | 25 ++ tests/functional/conftest.py | 363 ++++++--------------- tests/functional/utils/hooks.py | 392 +++++++++++++++++++++-- tests/functional/utils/test_framework.py | 2 +- 4 files changed, 488 insertions(+), 294 deletions(-) diff --git a/tests/functional/config.py b/tests/functional/config.py index 0874817cd4..b3b53d82d4 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -294,6 +294,25 @@ def get_uses_mapping(): """ TT_OVMS_IMAGE_LOCAL - ovms image can only be found locally """ ovms_image_local = get_bool("TT_OVMS_IMAGE_LOCAL", False) +""" TT_REQUIREMENTS - Requirements """ +req_ids = get_list("TT_REQUIREMENTS") + +""" TT_EXCLUDE_REQUIREMENTS - Requirements to exclude """ +exclude_req_ids = get_list("TT_EXCLUDE_REQUIREMENTS") + +""" TT_COMPONENTS - Components """ +components_ids = get_list("TT_COMPONENTS") + +""" TT_EXCLUDE_COMPONENTS - Components to exclude """ +exclude_components_ids = get_list("TT_EXCLUDE_COMPONENTS") + +""" TT_TESTS_PRIORITY_LIST - tests priority to run - high, medium or low """ +tests_priority_list_raw = get_list("TT_TESTS_PRIORITY_LIST", fallback=["high", "medium", "low"]) +tests_priority_list = [f"priority_{p}" for p in tests_priority_list_raw if "priority" not in p] + +""" TT_PERFORMANCE_TEST_TIMEOUT_MINUTES - timeout (in minutes) for each performance test """ +performance_test_timeout_minutes = get_int("TT_PERFORMANCE_TEST_TIMEOUT_MINUTES", 10) + """ TT_BASE_OS - os type used for calculating ovms_image name (if not given explicitly). Possible options (case insensitive): ubuntu22 - use default Ubuntu 22.04 image @@ -393,3 +412,9 @@ def get_ovms_types(): """ TT_OVMS_TYPE - ovms type runtime to be executed: DOCKER, BINARY, BINARY_DOCKER, CAPI, CAPI_DOCKER, DOCKER_CMD_LINE """ ovms_types = get_ovms_types() + +""" TT_DIVIDE_TARGET_DEVICE_PER_WORKER - spread tests across pytest workers based on target device """ +divide_target_device_per_worker = get_bool("TT_DIVIDE_TARGET_DEVICE_PER_WORKER", False) + +""" TT_PYTEST_KEYWORD_FILTER """ +pytest_keyword_filter = os.environ.get("TT_PYTEST_KEYWORD_FILTER", None) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index a910c29c3d..626e2cfbf5 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -14,29 +14,21 @@ # limitations under the License. # -import logging -import os +import random import sys import time -from logging import FileHandler - import pytest -from _pytest._code import ExceptionInfo, filter_traceback # noqa -from _pytest.outcomes import OutcomeException -from tests.functional.config import test_dir, test_dir_cleanup, artifacts_dir, enable_pytest_plugins +from tests.functional.config import enable_pytest_plugins, pytest_keyword_filter from tests.functional.constants.components import OvmsComponents -from tests.functional.constants.os_type import OsType from tests.functional.constants.ovms import ( BASE_OS_PARAM_NAME, + CURRENT_TARGET_DEVICE_DICT_ARGUMENT, OVMS_TYPE_PARAM_NAME, TARGET_DEVICE_PARAM_NAME, + TMP_REPOS_DIR_ARGUMENT, USES_MAPPING_PARAM_NAME, ) -from tests.functional.constants.ovms_binaries import calculate_ovms_binary_name -from tests.functional.constants.ovms_images import calculate_ovms_image_name -from tests.functional.constants.ovms_type import OvmsType -from tests.functional.constants.target_device import TargetDevice from tests.functional.utils import hooks from tests.functional.utils.hooks import ( log_configuration_variables, @@ -53,10 +45,11 @@ parametrize_uses_mapping, validate_port_pool, ) -from tests.functional.utils.marks import MarksRegistry, MarkRunType, MarkTestParameters +from tests.functional.utils.logger import OvmsFileHandler, get_logger +from tests.functional.utils.marks import MarksRegistry, MarkTestParameters from tests.functional.utils.test_framework import is_xdist_master -logger = logging.getLogger(__name__) +logger = get_logger(__name__) if enable_pytest_plugins: @@ -100,132 +93,41 @@ def pytest_configure(config): hooks.build_local_resources() hooks.validate_lock_files() hooks.list_host_zombie_processes() - # else: # Xdist worker thread - # hooks_utils.download_docker_images() - # hooks_utils.init_ovms_config_retrieved_from_master(config) + else: # Xdist worker thread + hooks.download_docker_images() + hooks.init_ovms_config_retrieved_from_master(config) - # hooks_utils.setup_nginx() + hooks.setup_nginx() # Let know that pytest was successfully configured config.configured = True - @pytest.hookimpl(hookwrapper=True) - def pytest_collection_modifyitems(session, config, items): - """ - Support for running tests with component tags. - Report all test component markers to mongo_reporter. - """ - logger.info("Preparing tests for test session in the following folder: {}".format(session.startdir)) - yield # deselect items in default hook way via keyword ('-k') - - # if config.option.collectonly: - # hooks_utils.log_skip_statistic(items) - - # deselected, all_components, all_requirements = preprocess_collected_items(items) - - # if deselected: - # hooks_utils.deselect_items(items, config, deselected) - - # random.Random(7).shuffle(items) - - - def preprocess_collected_items(items): - deselected = [] - all_components = {} - all_requirements = {} - # try: - # # required_marker_ids, excluded_marker_ids = get_marker_ids_for_test_run() - # for item in items: - # if getattr(item, "callspec", None): - # # Store calculated image for later use. - # ovms_type = item.callspec.params.get(OVMS_TYPE_PARAM_NAME, OvmsType.DOCKER) - # base_os = item.callspec.params.get(BASE_OS_PARAM_NAME, OsType.Ubuntu22) - # if ovms_type == OvmsType.BINARY or ovms_type == OvmsType.CAPI: - # item._image = calculate_ovms_binary_name(base_os=base_os) - # else: - # target_device = item.callspec.params.get( - # TARGET_DEVICE_PARAM_NAME, - # TargetDevice.TARGET_DEVICE_CPU, - # ) - # item._image = calculate_ovms_image_name(target_device=target_device, base_os=base_os) - # # add_dynamic_mark(item) - # test_type = MarkRunType.get_test_type_mark(item) - # # set_timeout_per_test_type(item, test_type) - # # update_parent_markers(item, MarkGeneral.COMPONENTS.mark) - # # update_parent_markers(item, MarkGeneral.REQIDS.mark) - # # update_parent_markers(item, MarkPriority.HIGH.mark) - # # update_parent_markers(item, MarkPriority.MEDIUM.mark) - # # update_parent_markers(item, MarkPriority.LOW.mark) - # # if deselect(item, test_type, required_marker_ids, excluded_marker_ids): - # # deselected.append(item) - # # continue - # # update_markers(item, test_type, all_components, MarkGeneral.COMPONENTS.mark) - # # update_markers(item, test_type, all_requirements, MarkGeneral.REQIDS.mark) - # - # except RuntimeError as e: - # error_msg = str(e) - # logger.exception(error_msg) - # sys.exit(error_msg) - # - # return deselected, all_components, all_requirements - - # def pytest_sessionfinish(session, exitstatus): - # current_test_run = hooks_utils.get_current_test_run() - # logger.info( - # "Finishing test session for test run type: {} in the following folder: {}".format( - # current_test_run, session.startdir - # ) - # ) - # test_status_report_header = f"{SEPARATOR} TEST TYPE STATUS REPORT - BEGIN {SEPARATOR}" - # logger.info(test_status_report_header) - # - # data_to_save = hooks_utils.collect_test_status_data(exitstatus) - # hooks_utils.save_test_data(data_to_save) - # - # test_status_report_footer = f"{SEPARATOR} TEST TYPE STATUS REPORT - END {SEPARATOR}" - # logger.info(test_status_report_footer) - # logger.info("Exit status is: {}".format(str(exitstatus))) - # - # if current_test_run != "": - # hooks_utils.get_test_run_reporter(current_test_run).on_run_end() - # else: - # for reporter in test_run_reporters.values(): # Finish all reporters - # reporter.on_run_end() - - - # def pytest_unconfigure(config): - # if getattr(config, "configured", None) is not True: - # # Check if pytest_configure() was done successfuly, if not: logger would be in invalid state so disable. - # for _logger in logger.manager.loggerDict.values(): - # _logger.disabled = True - # - # try: - # if is_xdist_master(): - # if restler_generate_evidence: - # restler_evidence_name_suffix = f"{release_product_version}_{datetime.now().strftime('%Y%m%d%H%M%S')}" - # shutil.make_archive(f"{restler_evidence_dir}_{restler_evidence_name_suffix}", "zip", - # restler_evidence_dir) - # hooks_utils.remove_ports_reservation(config) - # try: - # shutil.rmtree(config.tmp_repos_dir) - # except PermissionError as e: - # if all([ - # "C:\\" in config.tmp_repos_dir, - # type(e) == PermissionError - # ]): - # # workaround for Windows: https://jira.devtools.intel.com/browse/CVS-161953 - # change_dir_permissions(config.tmp_repos_dir) - # shutil.rmtree(config.tmp_repos_dir) - # hooks.teardown_environment() - # if machine_is_reserved_for_test_session: - # hooks_utils.clear_lockfiles() - # except Exception as e: - # error_msg = str(e) - # print(error_msg) - # sys.exit(error_msg) - -MarksRegistry.MARK_ENUMS.extend([OvmsComponents]) + def pytest_unconfigure(config): + if getattr(config, "configured", None) is not True: + # Check if pytest_configure() was done successfuly, if not: logger would be in invalid state so disable. + for _logger in logger.manager.loggerDict.values(): + _logger.disabled = True + + try: + if is_xdist_master(): + hooks.remove_ports_reservation(config) + hooks.cleanup_tmp_repos_dir(config) + hooks.teardown_environment() + if config.machine_is_reserved_for_test_session: + hooks.clear_lockfiles() + except Exception as e: + error_msg = str(e) + print(error_msg) + sys.exit(error_msg) + + + def pytest_configure_node(node): + node.workerinput[TMP_REPOS_DIR_ARGUMENT] = node.config.tmp_repos_dir + node.workerinput[CURRENT_TARGET_DEVICE_DICT_ARGUMENT] = node.config.current_target_device_dict + + + MarksRegistry.MARK_ENUMS.extend([OvmsComponents]) def pytest_sessionstart(session): @@ -234,6 +136,69 @@ def pytest_sessionstart(session): session.start_time = time.time() +# https://docs.pytest.org/en/6.2.x/reference.html#id57 +@pytest.hookimpl(hookwrapper=True) +def pytest_collection_modifyitems(session, config, items): + """ + Support for running tests with component tags. + Report all test component markers to mongo_reporter. + """ + logger.info("Preparing tests for test session in the following folder: {}".format(session.startdir)) + + if pytest_keyword_filter: + # Filter case insensitive + deselected = [_item for _item in items if pytest_keyword_filter.lower() not in _item.name.lower()] + if deselected: + hooks.deselect_items(items, config, deselected) + + yield # deselect items in default hook way via keyword ('-k') + + if config.option.collectonly: + hooks.log_skip_statistic(items) + + deselected = hooks.preprocess_collected_items(items) + if deselected: + hooks.deselect_items(items, config, deselected) + + hooks.set_divide_target_device_per_worker(items) + + random.Random(7).shuffle(items) + + +# https://docs.pytest.org/en/6.2.x/reference.html#id58 +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_protocol(item: "Item", nextitem: "Optional[Item]"): + """ + Perform the runtest protocol for a single test item. + The default runtest protocol is this (see individual hooks for full details): + pytest_runtest_logstart(nodeid, location) + Setup phase: + call = pytest_runtest_setup(item) (wrapped in CallInfo(when="setup")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + Call phase, if the setup passed and the setuponly pytest option is not set: + call = pytest_runtest_call(item) (wrapped in CallInfo(when="call")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + Teardown phase: + call = pytest_runtest_teardown(item, nextitem) (wrapped in CallInfo(when="teardown")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + pytest_runtest_logfinish(nodeid, location) + """ + __root_logger = get_logger(None) + if not item.keywords.get("skip"): + fh = OvmsFileHandler(item) + __root_logger.addHandler(fh) + yield + if not item.keywords.get("skip"): + fh.close() + __root_logger.removeHandler(fh) + + def pytest_generate_tests(metafunc): if OVMS_TYPE_PARAM_NAME in metafunc.fixturenames: parametrize_ovms_type(metafunc) @@ -261,133 +226,3 @@ def pytest_generate_tests(metafunc): if MarkTestParameters.MODEL_AUX_TYPE in metafunc.fixturenames: parametrize_model_aux_type(metafunc) - - #pytest_runtest_protocol ? move - - # - # def pytest_configure(): - # # Perform initial configuration. - # init_logger() - # - # init_conf_logger = logging.getLogger("init_conf") - # - # container_names = get_containers_with_tests_suffix() - # if container_names: - # init_conf_logger.info("Possible conflicting container names: {} " - # "for given tests_suffix: {}".format(container_names, get_tests_suffix())) - # - # if artifacts_dir: - # os.makedirs(artifacts_dir, exist_ok=True) - # - # - # def pytest_keyboard_interrupt(excinfo): - # clean_hanging_docker_resources() - # Server.stop_all_instances() - # - # - # def pytest_unconfigure(): - # # Perform cleanup. - # cleanup_logger = logging.getLogger("cleanup") - # - # cleanup_logger.info("Cleaning hanging docker resources with suffix: {}".format(get_tests_suffix())) - # clean_hanging_docker_resources() - # - # if test_dir_cleanup: - # cleanup_logger.info("Deleting test directory: {}".format(test_dir)) - # delete_test_directory() - # - # if len(Server.running_instances) > 0: - # logger.warning("Test got unstopped docker instances") - # Server.stop_all_instances() - # - # - # @pytest.hookimpl(hookwrapper=True) - # def pytest_collection_modifyitems(session, config, items): - # yield - # items = reorder_items_by_fixtures_used(session) - # - # - # @pytest.hookimpl(tryfirst=True, hookwrapper=True) - # def pytest_runtest_makereport(item, call): - # outcome = yield - # if call.when == "setup": - # report = outcome.get_result() - # report.test_metadata = {"start": call.start} - # - # - # @pytest.hookimpl(hookwrapper=True) - # def pytest_runtest_call(): - # __tracebackhide__ = True - # try: - # outcome = yield - # finally: - # pass - # exception_catcher("call", outcome) - # - # - # @pytest.hookimpl(hookwrapper=True) - # def pytest_runtest_setup(): - # __tracebackhide__ = True - # try: - # outcome = yield - # finally: - # pass - # exception_catcher("setup", outcome) - # - # - # @pytest.hookimpl(hookwrapper=True) - # def pytest_runtest_teardown(): - # __tracebackhide__ = True - # try: - # outcome = yield - # finally: - # pass - # exception_catcher("teardown", outcome) - # - # - # @pytest.hookimpl(hookwrapper=True) - # def pytest_runtest_teardown(item): - # yield - # # Test finished: remove test item for all fixtures that was used - # for fixture in item._server_fixtures: - # if item in item.session._server_fixtures_to_tests[fixture]: - # item.session._server_fixtures_to_tests[fixture].remove(item) - # if len(item.session._server_fixtures_to_tests[fixture]) == 0: - # # No other tests will use this docker instance so we can close it. - # Server.stop_by_fixture_name(fixture) - # - # - # def exception_catcher(when: str, outcome): - # if isinstance(outcome.excinfo, tuple): - # if len(outcome.excinfo) > 1 and isinstance(outcome.excinfo[1], OutcomeException): - # return - # exception_logger = logging.getLogger("exception_logger") - # exception_info = ExceptionInfo.from_exc_info(outcome.excinfo) - # exception_info.traceback = exception_info.traceback.filter(filter_traceback) - # exc_repr = exception_info.getrepr(style="short", chain=False)\ - # if exception_info.traceback\ - # else exception_info.exconly() - # exception_logger.error('Unhandled Exception during {}: \n{}' - # .format(when.capitalize(), str(exc_repr))) - # - # - # @pytest.hookimpl(hookwrapper=True, tryfirst=True) - # def pytest_runtest_logstart(nodeid, location): - # if artifacts_dir: - # test_name = get_path_friendly_test_name(location) - # log_path = os.path.join(artifacts_dir, f"{test_name}.log") - # _root_logger = logging.getLogger(None) - # _root_logger._test_log_handler = FileHandler(log_path) - # formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s") - # _root_logger._test_log_handler.setFormatter(formatter) - # _root_logger.addHandler(_root_logger._test_log_handler) - # yield - # - # - # @pytest.hookimpl(hookwrapper=True, tryfirst=True) - # def pytest_runtest_logfinish(nodeid, location): - # if artifacts_dir: - # _root_logger = logging.getLogger(None) - # _root_logger.removeHandler(_root_logger._test_log_handler) - # yield - diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index 44e7d8cc06..ea5edb25de 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -16,13 +16,19 @@ import itertools import os +import re import shutil import sys import warnings +import pytest -from collections import defaultdict +from collections import Counter, defaultdict, namedtuple from docker import errors as docker_errors +from itertools import groupby from pathlib import Path +from _pytest.mark import Mark, MarkDecorator +from _pytest.python import Function + from tests.functional import config from ovms.constants.models_library import ModelsLib from tests.functional.utils.download import wget_file @@ -32,6 +38,9 @@ build_test_image, c_api_wrapper_dir, cleanup_env_on_startup, + components_ids, + exclude_components_ids, + exclude_req_ids, force_generate_new_ssl_certs, global_tmp_dir_default, http_proxy, @@ -40,7 +49,12 @@ no_proxy, ovms_c_repo_path, ovms_file_locks_dir, + performance_test_timeout_minutes, machine_is_reserved_for_test_session, + req_ids, + run_ovms_with_opencl_trace, + run_ovms_with_valgrind, + tests_priority_list, tmp_dir, ) from tests.functional.constants.os_type import get_host_os, OsType, UBUNTU @@ -72,12 +86,19 @@ ) from tests.functional.constants.paths import Paths from tests.functional.constants.target_device import MAX_WORKERS_PER_TARGET_DEVICE, TargetDevice +from tests.functional.constants.ovms_binaries import calculate_ovms_binary_name from tests.functional.object_model.ovms_info import OvmsInfo from tests.functional.utils.core import TmpDir from tests.functional.utils.docker import DockerClient, DockerContainer, DOCKER_CONTAINER_TMP_PATH from tests.functional.utils.environment_info import EnvironmentInfo from tests.functional.utils.logger import get_logger -from tests.functional.utils.marks import MarkTestParameters +from tests.functional.utils.marks import ( + MarkConditionalRunType, + MarkGeneral, + MarkPriority, + MarkRunType, + MarkTestParameters, +) from tests.functional.utils.process import PID_STATE_ZOMBIE, Process, get_pid_name, get_pid_status from tests.functional.utils.test_framework import change_dir_permissions, get_test_object_prefix, is_xdist_master from tests.functional.object_model.ovsa import OvsaCerts @@ -85,10 +106,38 @@ logger = get_logger(__name__) +timeout_dict = defaultdict( + lambda: 5 * 60, + { + MarkRunType.TEST_MARK_ON_COMMIT: 3 * 60, + MarkRunType.TEST_MARK_REGRESSION: 5 * 60, + MarkRunType.TEST_MARK_REGRESSION_SINGLE: 5 * 60, + MarkRunType.TEST_MARK_REGRESSION_WEEKLY: 5 * 60, + MarkRunType.TEST_MARK_REGRESSION_WEEKLY_SINGLE: 5 * 60, + MarkRunType.TEST_MARK_ENABLING: 10 * 60, + MarkRunType.TEST_MARK_STRESS_AND_LOAD: 40 * 60, + MarkRunType.TEST_MARK_STRESS_AND_LOAD_SINGLE: 40 * 60, + MarkRunType.TEST_MARK_LONG: 48 * 60 * 60, + MarkRunType.TEST_MARK_SMOKE: 5 * 60, + MarkRunType.TEST_MARK_MANUAL: 5 * 60, + MarkRunType.TEST_MARK_PERFORMANCE: performance_test_timeout_minutes * 60, + }, +) + +TIMEOUT_MULTIPLIER: dict = { + TargetDevice.GPU: 1.5, + TargetDevice.NPU: 1.5, + "TRACE_TOOLS": 2, + "AUTO_HETERO_MULTI": 3, + OvmsType.KUBERNETES: 1.5, +} + CURRENT_TARGET_DEVICE_DICT = {} DEVICE_ID_TO_DETAILED_TARGET_DEVICE_NAME_MAP = defaultdict(lambda: ("", []), {}) +SkippedItem = namedtuple("SkippedItem", "test_name reason") + def init_environment(_config): global CURRENT_TARGET_DEVICE_DICT @@ -152,7 +201,7 @@ def cleanup_docker_images(): logger.info(f"Removed docker image: {image.id}") -def cleanup_tmp_repos_dir(): +def cleanup_tmp_repos_dir(config): try: shutil.rmtree(config.tmp_repos_dir) except PermissionError as e: @@ -735,30 +784,315 @@ def clear_lockfiles(): file.unlink() -def parametrize_tests(metafunc): - if OVMS_TYPE_PARAM_NAME in metafunc.fixturenames: - parametrize_ovms_type(metafunc) - - if USES_MAPPING_PARAM_NAME in metafunc.fixturenames: - parametrize_uses_mapping(metafunc) - - if BASE_OS_PARAM_NAME in metafunc.fixturenames: - parametrize_base_os(metafunc) - - if MarkTestParameters.MODEL_TYPE in metafunc.fixturenames: - parametrize_model_type(metafunc) - elif MarkTestParameters.ALL_MODELS in metafunc.fixturenames: - parametrize_all_models(metafunc) - elif MarkTestParameters.MANY_MODELS in metafunc.fixturenames: - parametrize_many_models(metafunc) - elif MarkTestParameters.ITERATION_INFO in metafunc.fixturenames: - parametrize_iteration_info(metafunc) - elif MarkTestParameters.INPUT_SHAPE in metafunc.fixturenames: - parametrize_input_shape(metafunc) - elif MarkTestParameters.PLUGIN_CONFIG in metafunc.fixturenames: - parametrize_plugin_config(metafunc) - elif TARGET_DEVICE_PARAM_NAME in metafunc.fixturenames: - parametrize_target_device(metafunc) +def deselect_items(items, config, deselected): + config.hook.pytest_deselected(items=deselected) + for item in deselected: + test_name = item.parent.nodeid + # nodeid comes in a way: + # 1) test.py::TestClass::() + # 2) test.py:: + if test_name[-2:] == "()": + test_name = test_name[:-2] + elif "::" not in test_name: + test_name += "::" + + test_name += item.name + logger.debug("Deselecting test: " + test_name) + items.remove(item) + + +def set_divide_target_device_per_worker(items): + # Assign xdist_group per target_device so --dist loadgroup routes + # all tests for a given device to the same worker. + # Must be done before yield so xdist sees the markers during scheduling. + if config.divide_target_device_per_worker: + num_devices = len(config.target_devices) + if num_devices and config.xdist_workers > 0 and config.xdist_workers % num_devices != 0: + raise ValueError( + f"xdist_workers ({config.xdist_workers}) must be a multiple of " + f"the number of target devices ({num_devices}): {config.target_devices}" + ) + for item in items: + if hasattr(item, "callspec") and TARGET_DEVICE_PARAM_NAME in item.callspec.params: + td = item.callspec.params[TARGET_DEVICE_PARAM_NAME] + item.add_marker(pytest.mark.xdist_group(name=f"device_{td}")) + logger.debug(f"Assigned {item.nodeid} to xdist_group device_{td}") + + +def preprocess_collected_items(items): + deselected = [] + all_components = {} + all_requirements = {} + try: + required_marker_ids, excluded_marker_ids = get_marker_ids_for_test_run() + for item in items: + preprocess_collected_item( + item, + deselected, + all_components, + all_requirements, + required_marker_ids, + excluded_marker_ids, + ) + + except RuntimeError as e: + error_msg = str(e) + logger.exception(error_msg) + sys.exit(error_msg) + + return deselected + + +def get_marker_ids_for_test_run(): + # requirements + if req_ids and exclude_req_ids: + raise RuntimeError("Can't both include and exclude requirements!") + # components + if components_ids and exclude_components_ids: + raise RuntimeError("Can't both include and exclude components!") + + required_marker_ids = generate_marker_ids(req_ids, components_ids, tests_priority_list) + excluded_marker_ids = generate_marker_ids(exclude_req_ids, exclude_components_ids) + return required_marker_ids, excluded_marker_ids + + +def generate_marker_ids(*args): + ids_lists = [ids_list for ids_list in args if ids_list] + marker_ids = [] + if len(ids_lists) > 1: + marker_ids = list(itertools.product(*ids_lists)) + elif len(ids_lists) == 1: + marker_ids = [(id_value,) for id_value in ids_lists[0]] + return marker_ids + + +def preprocess_collected_item( + item, deselected, all_components, all_requirements, required_marker_ids, excluded_marker_ids +): + set_item_image_parameter(item) + apply_conditional_run_type_marks(item) + test_type = MarkRunType.get_test_type_mark(item) + set_timeout_per_test_type(item, test_type) + update_parent_markers( + item, ( + MarkGeneral.COMPONENTS.mark, + MarkGeneral.REQIDS.mark, + MarkPriority.HIGH.mark, + MarkPriority.MEDIUM.mark, + MarkPriority.LOW.mark, + ) + ) + if deselect(item, test_type, required_marker_ids, excluded_marker_ids): + deselected.append(item) + else: + update_markers(item, test_type, all_components, MarkGeneral.COMPONENTS.mark) + update_markers(item, test_type, all_requirements, MarkGeneral.REQIDS.mark) + return deselected + + +def set_item_image_parameter(item): + if getattr(item, "callspec", None): + # Store calculated image for later use. + ovms_type = item.callspec.params.get(OVMS_TYPE_PARAM_NAME, OvmsType.DOCKER) + base_os = item.callspec.params.get(BASE_OS_PARAM_NAME, OsType.Ubuntu22) + if ovms_type == OvmsType.BINARY or ovms_type == OvmsType.CAPI: + item._image = calculate_ovms_binary_name(base_os=base_os) + else: + target_device = item.callspec.params.get(TARGET_DEVICE_PARAM_NAME, TargetDevice.CPU) + item._image = calculate_ovms_image_name(target_device=target_device, base_os=base_os) + + +def apply_conditional_run_type_marks(item): + """Resolve conditional_run_type and conditional_run_type_by_model meta-markers. + + conditional_run_type: assigns single_mark when device+OS match, default_mark otherwise. + conditional_run_type_by_model: assigns mark based on model_type membership in model collections. + """ + params = getattr(getattr(item, 'callspec', None), 'params', {}) + + for marker in item.iter_markers(MarkConditionalRunType.CONDITIONAL_RUN_TYPE): + single_mark = marker.kwargs["single_mark"] + default_mark = marker.kwargs["default_mark"] + single_if_device = marker.kwargs.get("single_if_device") + single_if_os = marker.kwargs.get("single_if_os") + + device = params.get(TARGET_DEVICE_PARAM_NAME, "") + base_os = str(params.get(BASE_OS_PARAM_NAME, "")).lower() + + is_single = True + if single_if_device and device not in single_if_device: + is_single = False + if single_if_os and base_os not in single_if_os: + is_single = False + + mark_name = single_mark if is_single else default_mark + item.add_marker(getattr(pytest.mark, mark_name)) + return # only first conditional_run_type marker is applied + + for marker in item.iter_markers(MarkConditionalRunType.CONDITIONAL_RUN_TYPE_BY_MODEL): + model_type = params.get(MarkTestParameters.MODEL_TYPE) + if model_type is None: + continue + device = params.get(TARGET_DEVICE_PARAM_NAME, "") + for mark_name, model_collection in marker.kwargs.get("model_mark_map", {}).items(): + device_models = set(model_collection.get(device, [])) + if model_type in device_models: + item.add_marker(getattr(pytest.mark, mark_name)) + return + default_mark = marker.kwargs.get("default_mark") + if default_mark: + item.add_marker(getattr(pytest.mark, default_mark)) + return + + +def set_timeout_per_test_type(item, test_type): + if item.get_closest_marker("timeout") is None: + ovms_type = item.callspec.params.get(OVMS_TYPE_PARAM_NAME, None) + value = timeout_dict[test_type] + if ovms_type == OvmsType.KUBERNETES: + value *= TIMEOUT_MULTIPLIER[OvmsType.KUBERNETES] + if any([test_type == MarkRunType.TEST_MARK_REGRESSION, + test_type == MarkRunType.TEST_MARK_ON_COMMIT, + ]): + if any(["AUTO" in item.name, "HETERO" in item.name, "MULTI" in item.name]): + value *= TIMEOUT_MULTIPLIER["AUTO_HETERO_MULTI"] + elif TargetDevice.GPU in item.name: + value *= TIMEOUT_MULTIPLIER[TargetDevice.GPU] + elif TargetDevice.NPU in item.name: + value *= TIMEOUT_MULTIPLIER[TargetDevice.NPU] + if run_ovms_with_valgrind or run_ovms_with_opencl_trace: + value *= TIMEOUT_MULTIPLIER["TRACE_TOOLS"] + item.add_marker(pytest.mark.timeout(value)) + + +def update_parent_markers(item, marker_types): + for marker_type in marker_types: + components = item.get_closest_marker(marker_type) + if components is not None: + current_components = next( + (component for component in item.own_markers if component.name == marker_type), + None, + ) + if current_components is None: + item.own_markers.append(components) + + +def deselect(item, test_type, required_marker_ids, excluded_marker_ids): + # Validate different scenarios where test should be deselected from execution during `collect` stage. + if isinstance(item, Function): + if test_type is None: + raise RuntimeError("Test do not have test_type: " + item.name) + + if required_marker_ids: + for required_marker_id_list in required_marker_ids: + if _is_test_marker_id_is_matched_with_id(item, required_marker_id_list): + # make sure that item is not deselected by other marker + return deselect_by_excluded_marker_ids(item, excluded_marker_ids) + return True + elif excluded_marker_ids: + return deselect_by_excluded_marker_ids(item, excluded_marker_ids) + + return False + + +def deselect_by_excluded_marker_ids(item, excluded_marker_ids): + for excluded_marker_ids_list in excluded_marker_ids: + if _is_test_marker_id_is_matched_with_id(item, excluded_marker_ids_list): + return True + return False + + +def _is_test_marker_id_is_matched_with_id(test, ids_to_check: list): + markers_to_check = [] + + for marker in test.own_markers: + if any([ + marker.name is MarkGeneral.REQIDS.value, + marker.name is MarkGeneral.COMPONENTS.value, + marker.name is MarkPriority.HIGH.mark, + marker.name is MarkPriority.MEDIUM.mark, + marker.name is MarkPriority.LOW.mark, + ]): + if marker.args: + for marker_arg in marker.args: + if isinstance(marker_arg, dict): + for param in marker_arg: + if param is None: + markers_to_check.append(str(marker_arg.values)) + elif param in test.name: + markers_to_check.append(str(marker_arg.values())) + elif isinstance(marker_arg, str): + markers_to_check.append(marker_arg) + else: + raise RuntimeError( + f"Test {test.name} do not have mark in correct form. Form: {type(marker_arg)}" + ) + else: + markers_to_check.append(marker.name) + + check_list = [] + for id_to_check in ids_to_check: + check_list.append(any(id_to_check.lower() in marker_to_check.lower() for marker_to_check in markers_to_check)) + + return all(check_list) + + +def update_markers(item, test_type, markers, marker_type): + marker = item.get_closest_marker(marker_type) + if marker is not None: + if test_type not in markers: + markers[test_type] = set() + markers[test_type].update(set(marker.args)) + + +def get_skipped_items(items): + skipped_items = [item for item in items if item.keywords.get("skip") is not None] + items = [] + for item in skipped_items: + skip_info = item.keywords.get("skip") + if isinstance(skip_info, (Mark, MarkDecorator)): + if "reason" in skip_info.kwargs: + reason = skip_info.kwargs["reason"] + elif skip_info.args: + reason = skip_info.args[0] + else: + reason = "" + items.append(SkippedItem(item.nodeid, reason)) + return items + + +def calc_statistics(items): + skipped_items = get_skipped_items(items) + issue_numbers = [] + other_tests = [] + for item in skipped_items: + match = re.search(r"DPNG-\d+", item.reason) + if match: + issue_numbers.append(match.group(0)) + else: + issue_numbers.append("others") + other_tests.append(item) + return Counter(issue_numbers), other_tests + + +def log_labeled_stats(issues): + msg = ["Skipped tests statistic:"] + issues_sorted_by_quantity = sorted(issues.items(), key=lambda i: i[1], reverse=True) + for issue, quantity in issues_sorted_by_quantity: + msg.append("{:>11}: {:>6}".format(issue, quantity)) + logger.info("\n".join(msg)) + + +def log_others(other_items): + msg = ["Skipped tests not labeled with issue:"] + items_grouped_by_reason = groupby(other_items, key=lambda i: i.reason) + for reason, items in list(items_grouped_by_reason): + msg.append("{}:".format(reason)) + msg.extend("|---{}".format(item.test_name) for item in list(items)) + logger.info("\n".join(msg)) + - if MarkTestParameters.MODEL_AUX_TYPE in metafunc.fixturenames: - parametrize_model_aux_type(metafunc) +def log_skip_statistic(items): + issue_stats, other_tests = calc_statistics(items) + log_labeled_stats(issue_stats) + log_others(other_tests) diff --git a/tests/functional/utils/test_framework.py b/tests/functional/utils/test_framework.py index 2e44c01a0e..745ee0e250 100644 --- a/tests/functional/utils/test_framework.py +++ b/tests/functional/utils/test_framework.py @@ -280,7 +280,7 @@ def _make_path_writable_and_retry(func, path, _exc_info): def remove_dir_tree(dir_path, ignore_errors=False): """Remove a directory tree, retrying failed paths after making them writable.""" try: - shutil.rmtree(dir_path, onexc=_make_path_writable_and_retry) + shutil.rmtree(dir_path, onerror=_make_path_writable_and_retry) except OSError: if not ignore_errors: raise From a427bd4c795d7fe1cb9a7cb3faab3aa166395998 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Thu, 18 Jun 2026 18:15:48 +0200 Subject: [PATCH 04/13] 18/06/2026 --- tests/functional/config.py | 5 ++ tests/functional/conftest.py | 4 +- tests/functional/constants/pipelines.py | 4 +- tests/functional/fixtures/ovms.py | 2 +- tests/functional/object_model/custom_node.py | 2 +- .../object_model/inference_helpers.py | 6 +- .../object_model/mediapipe_calculators.py | 2 +- tests/functional/object_model/ovms_capi.py | 2 +- tests/functional/object_model/ovms_config.py | 2 +- tests/functional/object_model/ovms_info.py | 3 +- .../functional/object_model/ovms_instance.py | 2 +- tests/functional/object_model/ovms_params.py | 5 +- .../python_custom_nodes.py | 90 ++++--------------- .../utils/generative_ai/validation_utils.py | 2 +- tests/functional/utils/hooks.py | 20 ++--- 15 files changed, 51 insertions(+), 100 deletions(-) diff --git a/tests/functional/config.py b/tests/functional/config.py index b3b53d82d4..0da9ebc9fe 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -90,6 +90,11 @@ def get_uses_mapping(): """ TT_DATASETS_PATH - Datasets local repo path""" datasets_path = get_path("TT_DATASETS_PATH", os.path.join("~", "ovms_datasets")) +""" TT_GENERATIVE_MODELS_LOCAL_PATH - local path for converted generative models """ +generative_models_local_path = get_path( + "TT_GENERATIVE_MODELS_LOCAL_PATH", os.path.join(models_path, "generative_models_ovms") +) + """ TT_CLEAN_ARTIFACTS_DIR """ clean_artifacts_dir = get_bool("TT_CLEAN_ARTIFACTS_DIR", False) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 626e2cfbf5..804573adb3 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -19,7 +19,7 @@ import time import pytest -from tests.functional.config import enable_pytest_plugins, pytest_keyword_filter +from tests.functional.config import enable_pytest_plugins, machine_is_reserved_for_test_session, pytest_keyword_filter from tests.functional.constants.components import OvmsComponents from tests.functional.constants.ovms import ( BASE_OS_PARAM_NAME, @@ -114,7 +114,7 @@ def pytest_unconfigure(config): hooks.remove_ports_reservation(config) hooks.cleanup_tmp_repos_dir(config) hooks.teardown_environment() - if config.machine_is_reserved_for_test_session: + if machine_is_reserved_for_test_session: hooks.clear_lockfiles() except Exception as e: error_msg = str(e) diff --git a/tests/functional/constants/pipelines.py b/tests/functional/constants/pipelines.py index fd2ba8588f..0881e0636d 100644 --- a/tests/functional/constants/pipelines.py +++ b/tests/functional/constants/pipelines.py @@ -24,7 +24,7 @@ from tests.functional.config import datasets_path from ovms.constants.model_dataset import RandomDataset -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from ovms.constants.models import ( AgeGender, ArgMax, @@ -1461,7 +1461,7 @@ def add_mediapipe_graphs_to_config(self, config, use_subconfig=False, mediapipe_ ) for i, model in enumerate(mediapipe_models): model_name = model.name - graph_name = model_name if (not model.is_llm and model.pbtxt_name is None) \ + graph_name = model_name if (not model.is_generative and model.pbtxt_name is None) \ else getattr(model, "pbtxt_name", None) graph_filename = f"{graph_name}.pbtxt" mediapipe_base_path = str(Path(Paths.MODELS_PATH_INTERNAL, model_name)) diff --git a/tests/functional/fixtures/ovms.py b/tests/functional/fixtures/ovms.py index c524929f99..1dfb4f91e5 100644 --- a/tests/functional/fixtures/ovms.py +++ b/tests/functional/fixtures/ovms.py @@ -38,7 +38,7 @@ run_ovms_with_opencl_trace, run_ovms_with_valgrind, ) -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.ovms import CurrentOvmsType, CurrentTarget from tests.functional.constants.ovms_binaries import calculate_ovms_binary_name from tests.functional.constants.ovms_images import ( diff --git a/tests/functional/object_model/custom_node.py b/tests/functional/object_model/custom_node.py index 6ffa56403f..cbf5f2efd5 100644 --- a/tests/functional/object_model/custom_node.py +++ b/tests/functional/object_model/custom_node.py @@ -25,7 +25,7 @@ from tests.functional.utils.logger import get_logger from tests.functional.utils.process import Process from tests.functional.config import custom_nodes_path, ovms_c_repo_path -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.ovms import CurrentOvmsType from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/inference_helpers.py b/tests/functional/object_model/inference_helpers.py index 782f6c1720..a78f4f0ad2 100644 --- a/tests/functional/object_model/inference_helpers.py +++ b/tests/functional/object_model/inference_helpers.py @@ -67,14 +67,14 @@ from tests.functional.utils.test_framework import FrameworkMessages, skip_if_runtime from tests.functional.utils.generative_ai.validation_utils import GenerativeAIValidationUtils from tests.functional.config import binary_io_images_path, wait_for_messages_timeout -from ovms.constants.model_dataset import ( +from tests.functional.models.models import ModelInfo +from tests.functional.models.models_datasets import ( BinaryDummyModelDataset, DefaultBinaryDataset, ExactShapeBinaryDataset, LanguageModelDataset, ModelDataset, ) -from ovms.constants.models import ModelInfo from tests.functional.constants.ovms import CurrentTarget as ct from tests.functional.constants.ovms import MediaPipeConstants, Ovms from tests.functional.constants.pipelines import SimpleMediaPipe @@ -818,7 +818,7 @@ def predict_and_assert(inference_infos: List[InferenceInfo], validate_results=Tr MediaPipeInferenceResponse.create(inference_info, outputs).validate( inference_info.input_data, output_key=output_key ) - elif inference_info.model.is_llm: + elif inference_info.model.is_generative: LLMInferenceResponse.create(inference_info, outputs).validate() else: InferenceResponse.create(inference_info, outputs).validate(inference_info.input_data) diff --git a/tests/functional/object_model/mediapipe_calculators.py b/tests/functional/object_model/mediapipe_calculators.py index 81f1eea0bf..50cd69fd8a 100644 --- a/tests/functional/object_model/mediapipe_calculators.py +++ b/tests/functional/object_model/mediapipe_calculators.py @@ -35,7 +35,7 @@ mediapipe_repo_branch, ovms_c_repo_path, ) -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.target_device import TargetDevice from tests.functional.constants.ovms import Config, MediaPipeConstants from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/ovms_capi.py b/tests/functional/object_model/ovms_capi.py index 010d33ac54..6726295155 100644 --- a/tests/functional/object_model/ovms_capi.py +++ b/tests/functional/object_model/ovms_capi.py @@ -33,7 +33,7 @@ from tests.functional.utils.test_framework import generate_test_object_name, skip_if_runtime from tests.functional.config import ovms_c_repo_path from tests.functional.constants.core import CONTAINER_STATUS_RUNNING -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.ovms import Config from tests.functional.constants.ovms_messages import OvmsMessages from tests.functional.constants.ovms_type import OvmsType diff --git a/tests/functional/object_model/ovms_config.py b/tests/functional/object_model/ovms_config.py index 88230c7576..8595d0771b 100644 --- a/tests/functional/object_model/ovms_config.py +++ b/tests/functional/object_model/ovms_config.py @@ -23,7 +23,7 @@ from tests.functional.constants.os_type import OsType from tests.functional.config import enable_plugin_config_target_device from tests.functional.constants.custom_loader import CustomLoaderConsts -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.ovms import Config, CurrentOvmsType, Ovms, set_plugin_config_boolean_value from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/ovms_info.py b/tests/functional/object_model/ovms_info.py index 237223118e..01c518e54b 100644 --- a/tests/functional/object_model/ovms_info.py +++ b/tests/functional/object_model/ovms_info.py @@ -28,6 +28,7 @@ from tests.functional.config import tmp_dir from tests.functional.constants.ovms_binaries import get_binaries, get_ovms_binary_cmd_setup from tests.functional.constants.ovms_type import OvmsType, OVMS_BINARY_PACKAGE_EXTENSIONS +from tests.functional.utils.docker import DockerClient logger = get_logger(__name__) @@ -213,8 +214,6 @@ def docker_pull_image_cli(image_to_pool): def pull_latest_image(cls, image_to_pull, force_pull=False): cls.docker_pull_image_cli(image_to_pull) # ensure image is available on host. - from tests.functional.utils.docker import DockerClient # pylint: disable=import-outside-toplevel - if image_to_pull not in cls.IMAGES or force_pull: repository, tag = image_to_pull.split(":") logger.info("Pulling image: {} tag: {}".format(repository, tag)) diff --git a/tests/functional/object_model/ovms_instance.py b/tests/functional/object_model/ovms_instance.py index 1f8ad15cfe..6ab0630a23 100644 --- a/tests/functional/object_model/ovms_instance.py +++ b/tests/functional/object_model/ovms_instance.py @@ -53,7 +53,7 @@ wait_for_messages_timeout, ) from tests.functional.constants.core import CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.target_device import MAX_WORKERS_PER_TARGET_DEVICE from tests.functional.constants.ovms import CurrentTarget as ct from tests.functional.constants.ovms import Ovms diff --git a/tests/functional/object_model/ovms_params.py b/tests/functional/object_model/ovms_params.py index 8102a54395..21e22cf449 100644 --- a/tests/functional/object_model/ovms_params.py +++ b/tests/functional/object_model/ovms_params.py @@ -25,8 +25,9 @@ from tests.functional.object_model.ovms_command import OvmsCommand from tests.functional.config import logging_level_ovms from tests.functional.constants.metrics import MetricsPolicy -from ovms.constants.models import ModelInfo, Muse -from ovms.constants.models_library import ModelsLib +from ovms.constants.models import Muse +from tests.functional.models.models import ModelInfo +from tests.functional.models.models_library import ModelsLib from tests.functional.object_model.cpu_extension import MuseModelExtension from tests.functional.object_model.custom_loader import CustomLoader diff --git a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py index bd5947f6ae..5b46500bcd 100644 --- a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py +++ b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py @@ -19,9 +19,9 @@ from tests.functional.utils.inference.communication import GRPC from tests.functional.utils.logger import get_logger from tests.functional.constants.generative_ai import GenerativeAIPluginConfig -from ovms.constants.model_dataset import LanguageModelDataset from tests.functional.constants.ovms import Ovms from tests.functional.constants.pipelines import MediaPipe, NodesConnection, NodeType, PythonGraphNode +from tests.functional.models.models_datasets import LanguageModelDataset from tests.functional.object_model.mediapipe_calculators import HttpLLMCalculator, PythonCalculator, \ ImageGenCalculator, EmbeddingsCalculatorOV, RerankCalculatorOV, S2tCalculator, T2sCalculator from tests.functional import config @@ -253,7 +253,7 @@ def is_pipeline(): return True -class SimpleLLM(SimplePythonCustomNodeMediaPipe): +class SimpleGenerativeNode(SimplePythonCustomNodeMediaPipe): inputs_number = 1 input_name: str = "input" outputs_number = 1 @@ -265,10 +265,11 @@ class SimpleLLM(SimplePythonCustomNodeMediaPipe): name: str = "" pbtxt_name: str = "simple_llm" is_python_custom_node: bool = True - is_llm: bool = True + is_generative: bool = True + is_llm: bool = False calculator_class = HttpLLMCalculator + use_subconfig: bool = False precision: str = None - allows_reasoning: bool = False def __init__(self, models_path, node_name="LLMExecutor", loopback=True, initialize_graphs=True, **kwargs): model = kwargs["model"] @@ -315,21 +316,16 @@ def __init__(self, models_path, node_name="LLMExecutor", loopback=True, initiali self.model_timeout = getattr(model, "model_timeout", None) -class SimpleImageGenerationLLM(SimpleLLM): - inputs_number = 1 - input_name: str = "input" - outputs_number = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True +class SimpleLLM(SimpleGenerativeNode): is_llm: bool = True + allows_reasoning: bool = False + + def __init__(self, models_path, node_name="LLMExecutor", loopback=False, initialize_graphs=True, **kwargs): + super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) + + +class SimpleImageGeneration(SimpleGenerativeNode): calculator_class = ImageGenCalculator - precision: str = None def __init__(self, models_path, node_name="ImageGenExecutor", initialize_graphs=True, **kwargs): model = kwargs["model"] @@ -368,61 +364,25 @@ def __init__(self, models_path, node_name="ImageGenExecutor", initialize_graphs= self.model_timeout = getattr(model, "model_timeout", None) -class SimpleFeatureExtractionLLM(SimpleLLM): - inputs_number: int = 1 - input_name: str = "input" - outputs_number: int = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True - is_llm: bool = True - use_subconfig: bool = True +class SimpleFeatureExtraction(SimpleGenerativeNode): is_feature_extraction: bool = True calculator_class = EmbeddingsCalculatorOV + use_subconfig: bool = True def __init__(self, models_path, node_name="LLMExecutor", loopback=False, initialize_graphs=True, **kwargs): super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) -class SimpleRerankLLM(SimpleLLM): - inputs_number: int = 1 - input_name: str = "input" - outputs_number: int = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True - is_llm: bool = True - use_subconfig: bool = True +class SimpleRerank(SimpleGenerativeNode): is_rerank: bool = True calculator_class = RerankCalculatorOV + use_subconfig: bool = True def __init__(self, models_path, node_name="LLMExecutor", loopback=False, initialize_graphs=True, **kwargs): super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) -class SimpleAsrModel(SimpleLLM): - inputs_number: int = 1 - input_name: str = "input" - outputs_number: int = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True - is_llm: bool = False +class SimpleAsrModel(SimpleGenerativeNode): is_asr_model: bool = True calculator_class = S2tCalculator @@ -430,19 +390,7 @@ def __init__(self, models_path, node_name="S2tExecutor", loopback=False, initial super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) -class SimpleTtsModel(SimpleLLM): - inputs_number: int = 1 - input_name: str = "input" - outputs_number: int = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True - is_llm: bool = False +class SimpleTtsModel(SimpleGenerativeNode): is_tts_model: bool = True calculator_class = T2sCalculator diff --git a/tests/functional/utils/generative_ai/validation_utils.py b/tests/functional/utils/generative_ai/validation_utils.py index 3cfbd55a04..07d728016f 100644 --- a/tests/functional/utils/generative_ai/validation_utils.py +++ b/tests/functional/utils/generative_ai/validation_utils.py @@ -33,7 +33,7 @@ from tests.functional.utils.inference.serving.openai import OpenAIWrapper, OpenAIFinishReason from tests.functional.config import save_image_to_artifacts from tests.functional.config import artifacts_dir, pipeline_type -from ovms.constants.model_dataset import FeatureExtractionModelDataset +from tests.functional.models.models_datasets import FeatureExtractionModelDataset logger = get_logger(__name__) diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index ea5edb25de..6d0f8c0ac6 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -30,7 +30,7 @@ from _pytest.python import Function from tests.functional import config -from ovms.constants.models_library import ModelsLib +from tests.functional.models.models_library import ModelsLib from tests.functional.utils.download import wget_file from tests.functional.utils.reservation_manager.args import parse_args from tests.functional.utils.reservation_manager.manager import Manager as ReservationManager @@ -428,14 +428,6 @@ def prepare_ovms_package(): def get_docker_images(images_to_download): - if OsType.Windows in config.base_os: - return - docker_ovms_types = [ - OvmsType.DOCKER, OvmsType.DOCKER_CMD_LINE, OvmsType.BINARY_DOCKER, OvmsType.CAPI_DOCKER - ] - if not any(_ovms_type in docker_ovms_types for _ovms_type in config.ovms_types): - return - images_to_download.add(config.minio_image) for target_device, base_os in itertools.product(config.target_devices, config.base_os): ovms_image = calculate_ovms_image_name(target_device, base_os) @@ -447,6 +439,12 @@ def get_docker_images(images_to_download): def download_docker_images(): + docker_ovms_types = [ + OvmsType.DOCKER, OvmsType.DOCKER_CMD_LINE, OvmsType.KUBERNETES, OvmsType.BINARY_DOCKER, OvmsType.CAPI_DOCKER + ] + if not any(_ovms_type in docker_ovms_types for _ovms_type in config.ovms_types): + return + images_to_download = set() images_to_download = get_docker_images(images_to_download) @@ -621,7 +619,7 @@ def parametrize_target_device(metafunc): def validate_lock_files(): """Ensure that target_device locks files exists""" - if not config.machine_is_reserved_for_test_session: + if not machine_is_reserved_for_test_session: return # Cannot validate locks validity since other testing session could acquire device lock locks = [value for key, value in vars(Paths).items() if "LOCK_FILE" in key] @@ -826,6 +824,7 @@ def preprocess_collected_items(items): try: required_marker_ids, excluded_marker_ids = get_marker_ids_for_test_run() for item in items: + set_item_image_parameter(item) preprocess_collected_item( item, deselected, @@ -869,7 +868,6 @@ def generate_marker_ids(*args): def preprocess_collected_item( item, deselected, all_components, all_requirements, required_marker_ids, excluded_marker_ids ): - set_item_image_parameter(item) apply_conditional_run_type_marks(item) test_type = MarkRunType.get_test_type_mark(item) set_timeout_per_test_type(item, test_type) From 8faab224cde08bca420da3aca72952e163ea1261 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Fri, 19 Jun 2026 08:39:11 +0200 Subject: [PATCH 05/13] 19/06/2026 --- tests/functional/constants/ovms_cohere.py | 25 +++ tests/functional/constants/requirements.py | 9 +- tests/functional/object_model/ovms_binary.py | 12 +- tests/functional/object_model/ovms_params.py | 9 +- tests/functional/test_embeddings.py | 113 +++++++++++ tests/functional/utils/generative_ai/utils.py | 191 ++++++++++++++++++ .../utils/generative_ai/validation_utils.py | 6 +- 7 files changed, 349 insertions(+), 16 deletions(-) create mode 100644 tests/functional/constants/ovms_cohere.py create mode 100644 tests/functional/test_embeddings.py create mode 100644 tests/functional/utils/generative_ai/utils.py diff --git a/tests/functional/constants/ovms_cohere.py b/tests/functional/constants/ovms_cohere.py new file mode 100644 index 0000000000..ad6c9ab030 --- /dev/null +++ b/tests/functional/constants/ovms_cohere.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from tests.functional.utils.inference.serving.cohere import CohereWrapper, OvmsRerankRequestParams + + +class OvmsCohereRequestParamsBuilder: + def __init__(self, endpoint, **kwargs): + self.endpoint = endpoint + if self.endpoint == CohereWrapper.RERANK: + self.request_params = OvmsRerankRequestParams(**kwargs) + else: + raise NotImplementedError diff --git a/tests/functional/constants/requirements.py b/tests/functional/constants/requirements.py index ecad4df3f0..1474b11129 100644 --- a/tests/functional/constants/requirements.py +++ b/tests/functional/constants/requirements.py @@ -39,7 +39,13 @@ class Requirements: kfservin_api = "CVS-81053 KFServing api" metrics = "CVS-43549 metrics" custom_nodes = "CVS-44359 custom nodes" + llm = "CVS-129298 LLM execution in ovms based on c++ code only" + embeddings_endpoint = "CVS-147460 embeddings endpoint" + rerank_endpoint = "CVS-147460 rerank endpoint" + images_endpoint = "CVS-169110 images endpoint" audio_endpoint = "CVS-174282 audio endpoint" + hf_imports = "CVS-162541 Direct models import from HF Hub in OVMS" + tools = "CVS-166514 Structured response with tools support in chat/completions" # test types sdl = "CVS-59335 SDL" @@ -62,7 +68,4 @@ class Requirements: streaming_api = "CVS-118064 streaming API extension" mediapipe = "CVS-103194 mediapipe" python_custom_node = "CVS-117210 python support" - llm = "CVS-129298 LLM execution in ovms based on c++ code only" openai_api = "CVS-138033 OpenAI API in OVMS" - hf_imports = "CVS-162541 Direct models import from HF Hub in OVMS" - tools = "CVS-166514 Structured response with tools support in chat/completions" diff --git a/tests/functional/object_model/ovms_binary.py b/tests/functional/object_model/ovms_binary.py index 7f61beca9c..adec1fdf5f 100644 --- a/tests/functional/object_model/ovms_binary.py +++ b/tests/functional/object_model/ovms_binary.py @@ -28,12 +28,10 @@ from tests.functional.config import artifacts_dir from tests.functional.constants.core import CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING -from ovms.constants.models import Muse from tests.functional.constants.ovms_binaries import get_ovms_binary_cmd_setup from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.paths import Paths from tests.functional.utils.log_monitor import LogMonitor -from tests.functional.object_model.cpu_extension import MuseModelExtension from tests.functional.object_model.mediapipe_calculators import MediaPipeCalculator from tests.functional.object_model.ovms_command import create_ovms_command from tests.functional.object_model.ovms_config import OvmsConfig @@ -100,10 +98,12 @@ def start_binary_ovms( MediaPipeCalculator.prepare_proto_calculator(parameters, config_dir_path_on_host, config_path_on_host) cpu_extension_path = None - if parameters.models is not None and any(isinstance(model, Muse) for model in parameters.models): - cpu_extension = MuseModelExtension() - cpu_extension_path = cpu_extension.lib_path[1:] - elif parameters.cpu_extension: + if parameters.models is not None: + for model in parameters.models: + if model.cpu_extension is not None: + cpu_extension = model.cpu_extension() + cpu_extension_path = cpu_extension.lib_path[1:] + if parameters.cpu_extension: if kwargs.get("replace_cpu_extension_params_for_binary", True): host_dir = os.path.join(resources_dir, Paths.CPU_EXTENSIONS) host_lib_path = os.path.join(host_dir, parameters.cpu_extension.lib_name) diff --git a/tests/functional/object_model/ovms_params.py b/tests/functional/object_model/ovms_params.py index 21e22cf449..857b2e09c0 100644 --- a/tests/functional/object_model/ovms_params.py +++ b/tests/functional/object_model/ovms_params.py @@ -25,10 +25,8 @@ from tests.functional.object_model.ovms_command import OvmsCommand from tests.functional.config import logging_level_ovms from tests.functional.constants.metrics import MetricsPolicy -from ovms.constants.models import Muse from tests.functional.models.models import ModelInfo from tests.functional.models.models_library import ModelsLib -from tests.functional.object_model.cpu_extension import MuseModelExtension from tests.functional.object_model.custom_loader import CustomLoader logger = get_logger(__name__) @@ -88,8 +86,11 @@ class OvmsParams(object): cache_size: int = None def __post_init__(self): - if self.models is not None and any(isinstance(model, Muse) for model in self.models): - self.cpu_extension = MuseModelExtension() + if self.models is not None: + for model in self.models: + if model.cpu_extension is not None: + self.cpu_extension = model.cpu_extension() + break def get_shape_param(self): result = self.shape diff --git a/tests/functional/test_embeddings.py b/tests/functional/test_embeddings.py new file mode 100644 index 0000000000..e64f6e07ef --- /dev/null +++ b/tests/functional/test_embeddings.py @@ -0,0 +1,113 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from tests.functional.models.models_library import ModelsLib +from tests.functional.constants.components import OvmsComponents +from tests.functional.constants.ovms_openai import EncodingFormatValues +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.requirements import Requirements +from tests.functional.constants.target_device import TargetDevice +from tests.functional.constants.target_device_configuration import nginx_mtls_not_supported_for_test +from tests.functional.object_model.inference_helpers import run_llm_inference +from tests.functional.utils.context import Context +from tests.functional.utils.generative_ai.utils import calculate_generative_test_timeout, GenerativeAIUtils +from tests.functional.utils.inference.serving.openai import OpenAIWrapper +from tests.functional.utils.logger import get_logger, step +from tests.functional.utils.test_framework import ( + skip_if_language_models_not_enabled, + skip_if_mediapipe_disabled, +) + +logger = get_logger(__name__) + + +@pytest.mark.priority_high +@pytest.mark.components(OvmsComponents.OVMS) +@pytest.mark.reqids(Requirements.embeddings_endpoint, Requirements.openai_api) +@pytest.mark.ovms_types_supported_for_test( + OvmsType.DOCKER, + OvmsType.DOCKER_CMD_LINE, + OvmsType.BINARY, + OvmsType.BINARY_DOCKER, +) +@skip_if_language_models_not_enabled() +@nginx_mtls_not_supported_for_test() +@skip_if_mediapipe_disabled() +class TestEmbeddings: + + @staticmethod + def run_embeddings_endpoints_test( + context: Context, model_type, openai_rest_api_type, endpoint, encoding_format, input_data_type + ): + model, result, port, request_params = GenerativeAIUtils.prepare_llm_resources( + context, + model_type, + openai_rest_api_type, + endpoint, + encoding_format=encoding_format, + ) + + step("Run simple inference") + run_llm_inference( + model, + openai_rest_api_type, + port, + endpoint, + input_data_type=input_data_type, + request_parameters=request_params, + ) + + GenerativeAIUtils.unload_llm_model_and_verify( + model, + result, + port, + openai_rest_api_type, + endpoint, + request_params + ) + + @pytest.mark.api_on_commit + @pytest.mark.devices_supported_for_test(TargetDevice.CPU, TargetDevice.GPU, TargetDevice.NPU) + @pytest.mark.model_type(ModelsLib.various_feature_extraction_models) + @pytest.mark.parametrize("endpoint", [OpenAIWrapper.EMBEDDINGS]) + @pytest.mark.parametrize("encoding_format", EncodingFormatValues.values(), ids=lambda x: f"encoding_format={x}") + @pytest.mark.parametrize("input_data_type", ["list", "string"], ids=lambda x: f"input_data_type={x}") + @pytest.mark.timeout(calculate_generative_test_timeout(480)) + def test_on_commit_embeddings_endpoints( + self, context: Context, model_type, openai_rest_api_type, endpoint, encoding_format, input_data_type + ): + """ + Description: + Execute single inference with LLM type model using embeddings endpoint. + + Input data: + - Language model (feature extraction) type + + Expected results: + OVMS will properly load language model and execute inference. + + Steps: + 1. Prepare language model instance + 2. Start OVMS + 3. Run simple inference + 4. Unload model + 5. Verify model is unreachable + """ + self.run_embeddings_endpoints_test( + context, model_type, openai_rest_api_type, endpoint, encoding_format, input_data_type + ) diff --git a/tests/functional/utils/generative_ai/utils.py b/tests/functional/utils/generative_ai/utils.py new file mode 100644 index 0000000000..c9bd1e0c56 --- /dev/null +++ b/tests/functional/utils/generative_ai/utils.py @@ -0,0 +1,191 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from openai import NotFoundError + +from tests.functional.config import ( + logging_level_ovms, + kv_cache_size_value, + kv_cache_precision_value, +) +from tests.functional.constants.generative_ai import GenerativeAIPluginConfig +from tests.functional.constants.ovms import CurrentTarget as ct +from tests.functional.constants.ovms_cohere import OvmsCohereRequestParamsBuilder +from tests.functional.constants.ovms_openai import OvmsOpenAIRequestParamsBuilder +from tests.functional.constants.ovms_messages import OvmsMessages +from tests.functional.fixtures.server import start_ovms +from tests.functional.object_model.inference_helpers import run_llm_inference +from tests.functional.object_model.ovms_params import OvmsParams +from tests.functional.utils.assertions import assert_raises_exception +from tests.functional.utils.context import Context +from tests.functional.utils.hooks import timeout_dict +from tests.functional.utils.inference.serving.cohere import CohereWrapper +from tests.functional.utils.logger import step +from tests.functional.utils.marks import MarkRunType +from tests.functional.object_model.python_custom_nodes.python_custom_nodes import ( + SimpleLLM, + SimpleFeatureExtraction, + SimpleRerank, + SimpleImageGeneration, + SimpleAsrModel, + SimpleTtsModel, +) + +INITIALIZE_LLM_TIMEOUT = 900 + + +def calculate_generative_test_timeout(lm_time_sec): + test_timeout_sec = lm_time_sec + INITIALIZE_LLM_TIMEOUT + return ( + test_timeout_sec + if test_timeout_sec > timeout_dict[MarkRunType.TEST_MARK_REGRESSION_WEEKLY_SINGLE] or ct.is_gpu_target() + else timeout_dict[MarkRunType.TEST_MARK_REGRESSION_WEEKLY_SINGLE] + ) + + +class GenerativeAIUtils: + + @staticmethod + def prepare_request_params(endpoint, **request_params_kwargs): + request_params_builder_class = OvmsCohereRequestParamsBuilder if endpoint == CohereWrapper.RERANK \ + else OvmsOpenAIRequestParamsBuilder + request_params_builder = request_params_builder_class( + endpoint=endpoint, + **request_params_kwargs + ) + request_params = request_params_builder.request_params + return request_params + + @staticmethod + def prepare_model( + model_type, + kv_cache_size=kv_cache_size_value, + plugin_config=None, + max_position_embeddings=None, + tools_enabled=None, + apply_gorilla_patch=False, + enable_tool_guided_generation=False, + target_device=None, + resolution=None, + apply_short_name=False, + **kwargs + ): + if plugin_config is None: + plugin_config = {GenerativeAIPluginConfig.KV_CACHE_PRECISION: kv_cache_precision_value} + + stream = kwargs.get("stream", False) + + step("Prepare language model instance") + llm = model_type() + llm.apply_gorilla_patch = apply_gorilla_patch + if apply_gorilla_patch: + llm.name = f"{llm.gorilla_patch_name}-stream" if stream else llm.gorilla_patch_name + if apply_short_name: + llm.name = llm.name.split("/")[-1] if "/" in llm.name else llm.name + if tools_enabled: + llm.tools_enabled = True + + node_name = "LLMExecutor" + if hasattr(llm, "is_feature_extraction") and llm.is_feature_extraction: + node_class = SimpleFeatureExtraction + elif hasattr(llm, "is_rerank") and llm.is_rerank: + node_class = SimpleRerank + elif hasattr(llm, "is_image_generation") and llm.is_image_generation: + node_class = SimpleImageGeneration + node_name = "ImageGenExecutor" + elif hasattr(llm, "is_asr_model") and llm.is_asr_model: + node_class = SimpleAsrModel + node_name = "S2tExecutor" + elif hasattr(llm, "is_tts_model") and llm.is_tts_model: + node_class = SimpleTtsModel + node_name = "T2sExecutor" + else: + node_class = SimpleLLM + + model = node_class( + model=llm, + node_name=node_name, + models_path=llm.model_path_on_host, + kv_cache_size=kv_cache_size, + plugin_config=plugin_config, + enable_tool_guided_generation=enable_tool_guided_generation, + target_device=target_device, + resolution=resolution, + ) + model.precision = llm.precision + model.max_position_embeddings = max_position_embeddings + model.jinja_template = llm.jinja_template + model.allows_reasoning = llm.allows_reasoning + model.apply_gorilla_patch = apply_gorilla_patch + model.gorilla_patch_name = llm.gorilla_patch_name + model.enable_tool_guided_generation = enable_tool_guided_generation + model.bfcl_num_threads = llm.bfcl_num_threads + return model + + @classmethod + def prepare_resources( + cls, context: Context, model_type, openai_rest_api_type, endpoint, log_level=logging_level_ovms, + kv_cache_size=kv_cache_size_value, plugin_config=None, max_position_embeddings=None, env=None, + allowed_local_media_path=None, allowed_media_domains=None, target_device=None, resolution=None, + apply_short_name=False, **request_params_kwargs, + ): + if plugin_config is None: + plugin_config = {GenerativeAIPluginConfig.KV_CACHE_PRECISION: kv_cache_precision_value} + + model = cls.prepare_model( + model_type, + kv_cache_size, + plugin_config, + max_position_embeddings, + target_device=target_device, + resolution=resolution, + apply_short_name=apply_short_name, + ) + + step("Start OVMS") + result = start_ovms( + context, + OvmsParams(models=[model], use_config=True, use_subconfig=model.use_subconfig, log_level=log_level, + allowed_local_media_path=allowed_local_media_path, allowed_media_domains=allowed_media_domains), + timeout=model.model_timeout, environment=env, + ) + port = result.ovms.get_port(openai_rest_api_type) + + request_params = cls.prepare_request_params(endpoint, **request_params_kwargs) + + return model, result, port, request_params + + @staticmethod + def unload_model_and_verify(model, result, port, openai_rest_api_type, endpoint, request_parameters, + dataset=None, error_type=NotFoundError): + step("Unload model") + ovms_log_monitor = result.ovms.create_log(False) + result.ovms.unload_all_models() + ovms_log_monitor.models_unloaded([model]) + result.models = [] + + step("Verify model is unreachable") + assert_raises_exception( + error_type, + OvmsMessages.MEDIAPIPE_IS_RETIRED, + run_llm_inference, + model=model, + api_type=openai_rest_api_type, + port=port, + endpoint=endpoint, + dataset=dataset, + request_parameters=request_parameters, + ) diff --git a/tests/functional/utils/generative_ai/validation_utils.py b/tests/functional/utils/generative_ai/validation_utils.py index 07d728016f..3c6565c1b4 100644 --- a/tests/functional/utils/generative_ai/validation_utils.py +++ b/tests/functional/utils/generative_ai/validation_utils.py @@ -34,6 +34,7 @@ from tests.functional.config import save_image_to_artifacts from tests.functional.config import artifacts_dir, pipeline_type from tests.functional.models.models_datasets import FeatureExtractionModelDataset +from tests.functional.utils.generative_ai.utils import GenerativeAIUtils logger = get_logger(__name__) @@ -545,7 +546,7 @@ def create_embeddings_getter( # pylint: disable=import-outside-toplevel api_type: OpenAI REST API type. port: OVMS port where the embeddings model is served. request_parameters: Optional pre-built request parameters for embeddings endpoint. - If None, will be built automatically via LLMUtils.prepare_request_params. + If None, will be built automatically via GenerativeAIUtils.prepare_request_params. inference_fn: Callable to run LLM inference (e.g. run_llm_inference). Injected to avoid circular import between this module and inference_helpers. @@ -557,8 +558,7 @@ def create_embeddings_getter( # pylint: disable=import-outside-toplevel ) if request_parameters is None: - from llm.utils import LLMUtils - request_parameters = LLMUtils.prepare_request_params(OpenAIWrapper.EMBEDDINGS) + request_parameters = GenerativeAIUtils.prepare_request_params(OpenAIWrapper.EMBEDDINGS) def getter(text): class TextDataset(FeatureExtractionModelDataset): From 9415fb6b5b6f286b6a3ee6eccb1f0adefabf1031 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Fri, 19 Jun 2026 17:14:04 +0200 Subject: [PATCH 06/13] 19/06/2026 part 2 --- tests/functional/config.py | 13 +++- tests/functional/conftest.py | 3 + .../python_custom_nodes.py | 8 +-- tests/functional/test_embeddings.py | 4 +- .../utils/generative_ai/validation_utils.py | 2 +- tests/functional/utils/hooks.py | 26 +++++++- tests/functional/utils/ov_hf_downloader.py | 65 +++++++++++++++++++ tests/requirements.txt | 39 +++++++---- 8 files changed, 136 insertions(+), 24 deletions(-) create mode 100644 tests/functional/utils/ov_hf_downloader.py diff --git a/tests/functional/config.py b/tests/functional/config.py index 0da9ebc9fe..1b02977d3c 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -20,8 +20,7 @@ from tests.functional.constants.os_type import OsType from tests.functional.constants.ovms_type import OvmsType -from tests.functional.constants.target_device import TargetDevice -from tests.functional.utils.core import TmpDir +from tests.functional.utils.core import TmpDir, get_token_value from tests.functional.utils.helpers import ( generate_test_object_name, get_bool, @@ -92,7 +91,7 @@ def get_uses_mapping(): """ TT_GENERATIVE_MODELS_LOCAL_PATH - local path for converted generative models """ generative_models_local_path = get_path( - "TT_GENERATIVE_MODELS_LOCAL_PATH", os.path.join(models_path, "generative_models_ovms") + "TT_GENERATIVE_MODELS_LOCAL_PATH", os.path.join(models_path, "generative_models") ) """ TT_CLEAN_ARTIFACTS_DIR """ @@ -423,3 +422,11 @@ def get_ovms_types(): """ TT_PYTEST_KEYWORD_FILTER """ pytest_keyword_filter = os.environ.get("TT_PYTEST_KEYWORD_FILTER", None) + +""" TT_HUGGINGFACE_TOKEN_FILE_PATH - path to file containing huggingface token """ +huggingface_token_file_path = get_path( + "TT_HUGGINGFACE_TOKEN_FILE_PATH", os.path.join("~", "ovms_tokens", "huggingface_token") +) + +""" TT_HUGGINGFACE_TOKEN - huggingface token value. Env var takes priority, then file. """ +huggingface_token = os.environ.get("TT_HUGGINGFACE_TOKEN") or get_token_value(huggingface_token_file_path, "") diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 804573adb3..b0626ef04c 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -53,6 +53,9 @@ if enable_pytest_plugins: + + raise NotImplementedError("OVMS tests not enabled") + pytest_plugins = [ "tests.functional.fixtures.ovms", "tests.functional.fixtures.server", diff --git a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py index 5b46500bcd..4646889f53 100644 --- a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py +++ b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py @@ -47,7 +47,7 @@ class SimplePythonCustomNodeMediaPipe(MediaPipe): def __init__(self, handler_path, node_name="upper_text", loopback=False, initialize_graphs=True, **kwargs): self.node_name = node_name self.config = {} - self.prepare_llm_model_inputs_outputs(model=self, dataset=LanguageModelDataset, kwargs=kwargs) + self.prepare_model_inputs_outputs(model=self, dataset=LanguageModelDataset, kwargs=kwargs) super().__init__(**kwargs) self.calculators = [ @@ -65,7 +65,7 @@ def __init__(self, handler_path, node_name="upper_text", loopback=False, initial self.handler_path = handler_path @staticmethod - def prepare_llm_model_inputs_outputs(model, dataset, **kwargs): + def prepare_model_inputs_outputs(model, dataset, **kwargs): inputs_number = kwargs.get("inputs_number", None) model.inputs_number = inputs_number if inputs_number is not None else model.inputs_number model.inputs = { @@ -276,7 +276,7 @@ def __init__(self, models_path, node_name="LLMExecutor", loopback=True, initiali self.node_name = node_name self.config = {} dataset = model.get_default_dataset() - self.prepare_llm_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) + self.prepare_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) self.regular_models = [] self.is_mediapipe = True @@ -332,7 +332,7 @@ def __init__(self, models_path, node_name="ImageGenExecutor", initialize_graphs= self.node_name = node_name self.config = {} dataset = model.get_default_dataset() - self.prepare_llm_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) + self.prepare_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) self.regular_models = [] self.is_mediapipe = True diff --git a/tests/functional/test_embeddings.py b/tests/functional/test_embeddings.py index e64f6e07ef..ae26ac515c 100644 --- a/tests/functional/test_embeddings.py +++ b/tests/functional/test_embeddings.py @@ -54,7 +54,7 @@ class TestEmbeddings: def run_embeddings_endpoints_test( context: Context, model_type, openai_rest_api_type, endpoint, encoding_format, input_data_type ): - model, result, port, request_params = GenerativeAIUtils.prepare_llm_resources( + model, result, port, request_params = GenerativeAIUtils.prepare_resources( context, model_type, openai_rest_api_type, @@ -72,7 +72,7 @@ def run_embeddings_endpoints_test( request_parameters=request_params, ) - GenerativeAIUtils.unload_llm_model_and_verify( + GenerativeAIUtils.unload_model_and_verify( model, result, port, diff --git a/tests/functional/utils/generative_ai/validation_utils.py b/tests/functional/utils/generative_ai/validation_utils.py index 3c6565c1b4..2ce038123d 100644 --- a/tests/functional/utils/generative_ai/validation_utils.py +++ b/tests/functional/utils/generative_ai/validation_utils.py @@ -34,7 +34,6 @@ from tests.functional.config import save_image_to_artifacts from tests.functional.config import artifacts_dir, pipeline_type from tests.functional.models.models_datasets import FeatureExtractionModelDataset -from tests.functional.utils.generative_ai.utils import GenerativeAIUtils logger = get_logger(__name__) @@ -558,6 +557,7 @@ def create_embeddings_getter( # pylint: disable=import-outside-toplevel ) if request_parameters is None: + from tests.functional.utils.generative_ai.utils import GenerativeAIUtils request_parameters = GenerativeAIUtils.prepare_request_params(OpenAIWrapper.EMBEDDINGS) def getter(text): diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index 6d0f8c0ac6..6e8fa6ec93 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -30,7 +30,7 @@ from _pytest.python import Function from tests.functional import config -from tests.functional.models.models_library import ModelsLib +from tests.functional.models.models_library import ModelsLib, ModelsLibrary from tests.functional.utils.download import wget_file from tests.functional.utils.reservation_manager.args import parse_args from tests.functional.utils.reservation_manager.manager import Manager as ReservationManager @@ -54,6 +54,7 @@ req_ids, run_ovms_with_opencl_trace, run_ovms_with_valgrind, + target_devices, tests_priority_list, tmp_dir, ) @@ -99,6 +100,7 @@ MarkRunType, MarkTestParameters, ) +from tests.functional.utils.ov_hf_downloader import OVHfDownloader from tests.functional.utils.process import PID_STATE_ZOMBIE, Process, get_pid_name, get_pid_status from tests.functional.utils.test_framework import change_dir_permissions, get_test_object_prefix, is_xdist_master from tests.functional.object_model.ovsa import OvsaCerts @@ -427,6 +429,26 @@ def prepare_ovms_package(): setup_capi_wrapper(package_content) +def get_models_to_download(): + models_to_download = [] + for various_models_name in [name for name, obj in vars(ModelsLibrary).items() if isinstance(obj, property)]: + various_models_value = getattr(ModelsLib, various_models_name) + if isinstance(various_models_value, dict): + for target_device in target_devices: + models_to_download.extend(various_models_value[target_device]) + else: + models_to_download.extend(various_models_value) + return list(set(models_to_download)) + + +def download_models(): + models_to_download = get_models_to_download() + for model_type in models_to_download: + if model_type.is_local: + ov_hf_downloader = OVHfDownloader(model_type) + ov_hf_downloader.check_and_update_hf_model() + + def get_docker_images(images_to_download): images_to_download.add(config.minio_image) for target_device, base_os in itertools.product(config.target_devices, config.base_os): @@ -455,7 +477,7 @@ def download_docker_images(): def download_resources_master(): print("Download required resources") - + download_models() download_docker_images() diff --git a/tests/functional/utils/ov_hf_downloader.py b/tests/functional/utils/ov_hf_downloader.py new file mode 100644 index 0000000000..55cd9f5750 --- /dev/null +++ b/tests/functional/utils/ov_hf_downloader.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from datetime import datetime, timezone +from huggingface_hub import HfApi, snapshot_download + +from tests.functional.config import huggingface_token +from tests.functional.utils.logger import get_logger +from tests.functional.utils.test_framework import get_dir_latest_mtime, remove_dir_tree, swap_directory + +logger = get_logger(__name__) + + +class OVHfDownloader: + + def __init__(self, model_type, model_base_path=None): + if not huggingface_token: + raise Exception( + "Provide huggingfacace_token with TT_HUGGINGFACE_TOKEN or TT_HUGGINGFACE_TOKEN_FILE_PATH envs" + ) + self.api = HfApi(token=huggingface_token) + self.model = model_type() + self.model_name = self.model.name + if model_base_path is None: + self.model_local_path = self.model.model_path_on_host + else: + self.model_local_path = os.path.join(model_base_path, self.model.name) + + def check_and_update_hf_model(self): + repo_info = self.api.repo_info(self.model_name) + local_latest_mtime = get_dir_latest_mtime(self.model_local_path) + local_latest_dt = datetime.fromtimestamp(local_latest_mtime, tz=timezone.utc) if local_latest_mtime else None + + if local_latest_dt and repo_info.last_modified <= local_latest_dt: + print(f"No files to update for model: {self.model_name}") + return + + print(f"Download OVHf model: {self.model_name}") + staging_path = self.model_local_path + "_staging" + if os.path.exists(staging_path): + remove_dir_tree(staging_path) + self.download_model(model_dir=staging_path) + swap_directory(self.model_local_path, staging_path) + + def download_model(self, model_name=None, model_dir=None, force_download=False): + snapshot_download( + repo_id=self.model_name if model_name is None else model_name, + local_dir=self.model_local_path if model_dir is None else model_dir, + token=huggingface_token, + force_download=force_download, + ) diff --git a/tests/requirements.txt b/tests/requirements.txt index c4424528ce..6b4c431f3d 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,15 +1,30 @@ -boto3==1.33.13 -checksec-py==0.7.4 +# supported python version: 3.12 +cohere==7.0.4 +dataclasses-json==0.6.7 +distro==1.9.0 docker==7.1.0 -grpcio==1.60.0 +filelock==3.29.4 +GitPython==3.1.50 +grpcio==1.81.1 +Jinja2==3.1.6 +jiwer>=4.0.0 +openai==2.43.0 +opencv-python==4.13.0.92 paramiko==5.0.0 -psutil==5.9.6 -pytest==9.0.3 -pytest-json==0.4.0 -tensorflow-serving-api>=2.16.1 -tensorflow>=2.16.1 -requests==2.33.0 +Pillow==12.2.0 +protobuf==7.35.1 +psutil==7.2.2 +pytest==9.1.0 +pytest-timeout==2.4.0 +pytest-xdist==3.8.0 +python-dateutil==2.8.2 +pyyaml==6.0.3 +requests==2.34.2 +requests-toolbelt==1.0.0 retry==0.9.2 -protobuf<=5.29.6 -jsonschema<=4.23.0 -openai<=1.84.0 +setuptools==81.0.0 +soundfile==0.14.0 +tensorboard==2.20.0 # why not in ovms-test +tensorflow==2.21.0 +tensorflow-serving-api==2.20.0 +tritonclient[all]==2.69.0 From 19924073aac8578cdafba023502f220cac67ac69 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Fri, 19 Jun 2026 17:18:17 +0200 Subject: [PATCH 07/13] 19/06/2026 part 3 --- tests/functional/models/__init__.py | 15 + tests/functional/models/models.py | 608 +++++++++++++++++++ tests/functional/models/models_datasets.py | 261 ++++++++ tests/functional/models/models_generative.py | 186 ++++++ tests/functional/models/models_library.py | 27 + 5 files changed, 1097 insertions(+) create mode 100644 tests/functional/models/__init__.py create mode 100644 tests/functional/models/models.py create mode 100644 tests/functional/models/models_datasets.py create mode 100644 tests/functional/models/models_generative.py create mode 100644 tests/functional/models/models_library.py diff --git a/tests/functional/models/__init__.py b/tests/functional/models/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/models/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/models/models.py b/tests/functional/models/models.py new file mode 100644 index 0000000000..09f1c17ec0 --- /dev/null +++ b/tests/functional/models/models.py @@ -0,0 +1,608 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import math +import os +import shutil +import stat +from dataclasses import dataclass +from distutils.dir_util import copy_tree, remove_tree +from enum import Enum +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import numpy as np + +from tests.functional.config import is_nginx_mtls, models_path, xdist_workers +from tests.functional.constants.os_type import OsType, get_host_os +from tests.functional.constants.ovms import Ovms +from tests.functional.constants.paths import Paths +from tests.functional.constants.target_device import TargetDevice +from tests.functional.object_model.cpu_extension import CpuExtension +from tests.functional.object_model.custom_loader import CustomLoader +from tests.functional.object_model.ovms_mapping_config import OvmsMappingConfig +from tests.functional.object_model.shape import Shape +from tests.functional.object_model.test_environment import TestEnvironment +from tests.functional.utils.logger import get_logger + +logger = get_logger(__name__) + + +class ModelType(str, Enum): + IR = "IR" + ONNX = "ONNX" + PDPD = "PDPD" + TFSM = "TFSM" # tensorflow savedmodel + + +CLOUD_HEADERS = {"azure-blob": "az://", "azure-fs": "azfs://", "google": "gs://", "s3_minio": "s3://"} + +DEVICE_LOADING_SPEED = { + TargetDevice.CPU: { + ModelType.IR: 97.4, + ModelType.ONNX: 58.1, + ModelType.PDPD: 97.4, + ModelType.TFSM: 97.4, + }, # For now TFSM works only with CPU. + TargetDevice.GPU: {ModelType.IR: 10, ModelType.ONNX: 25, ModelType.PDPD: 10}, + TargetDevice.NPU: {ModelType.IR: 10, ModelType.ONNX: 25, ModelType.PDPD: 10}, + TargetDevice.AUTO: {ModelType.IR: 2, ModelType.ONNX: 5, ModelType.PDPD: 2}, + TargetDevice.HETERO: {ModelType.IR: 2, ModelType.ONNX: 5, ModelType.PDPD: 2}, + TargetDevice.AUTO_CPU_GPU: {ModelType.IR: 2, ModelType.ONNX: 5, ModelType.PDPD: 2}, +} + + +@dataclass +class ModelInfo: + name: str = None + version: int = 1 + inputs: dict = None + outputs: dict = None + batch_size: int = None + model_version_policy: str = None + nireq: int = None + plugin_config: object = None + transpose_axes: str = None + custom_loader: CustomLoader = None + is_stateful: bool = False + expected_batch_size: int = None + max_sequence_number: int = None + idle_sequence_cleanup: bool = None + low_latency_transformation: bool = None + base_path: str = None + model_path_on_host = None + model_type: ModelType = ModelType.IR + input_shape_for_ovms: Any = None + _compiled_layout: str = None + _layout_for_ovms: str = None + allow_cache: str = None + use_mapping: bool = None + target_device: str = None # Currently used target device. Set up prior test in context fixture + base_os: str = None + is_mediapipe: bool = False + is_language: bool = False + use_relative_paths: bool = False + use_subconfig: bool = False + xml_name: str = None + onnx_name: str = None + is_llm: bool = False + is_vision_language: bool = False + is_hf_direct_load: bool = False # model can be loaded directly from HuggingFace without conversion with optimum-cli + gguf_filename: str = None + is_local: bool = False + model_subpath: str = None + single_mediapipe_model_mode: bool = False + tool_parser: str = None + jinja_template: str = None + tools_enabled: bool = False + apply_gorilla_patch: bool = False + gorilla_patch_name: str = None + is_agentic: bool = False + enable_tool_guided_generation: bool = False + reasoning_parser: str = None + pooling: str = None + extra_quantization_params: str = None + pipeline_type: str = None + bfcl_num_threads: int = None + max_num_batched_tokens: int = None + is_audio: bool = False + is_asr_model: bool = False + is_tts_model: bool = False + predict_timeout: Optional[int] = None + copy_all_model_versions: bool = False + cpu_extension: CpuExtension = None + + def __post_init__(self): + if self.use_relative_paths: + self.base_path = self.name + else: + if self.use_subconfig: + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, f"{self.name}_mediapipe", self.name) + else: + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, self.name) + self.model_path_on_host = os.path.join(models_path, self.name, str(self.version)) + self.set_additional_model_params() + if get_host_os() == OsType.Windows: + # enable model deletion https://jira.devtools.intel.com/browse/CVS-160412 + self.plugin_config = Ovms.PLUGIN_CONFIG_WINDOWS + + def set_additional_model_params(self): + if not self.base_os: + self.base_os = ModelInfo.base_os + if not self.target_device: + self.target_device = ModelInfo.target_device + + @staticmethod + def get_list_of_config_fields(): + return [ + "name", + "base_path", + "batch_size", + "model_version_policy", + "nireq", + "plugin_config", + "stateful", + "max_sequence_number", + "idle_sequence_cleanup", + "low_latency_transformation", + "allow_cache", + ] + + def get_model_file_path(self): + filepath = None + if self.model_type == ModelType.IR: + ext = "xml" + xml_name = self.xml_name if self.xml_name is not None else self.name + filepath = f"{models_path}/{self.name}/{self.version}/{xml_name}.{ext}" + elif self.model_type == ModelType.ONNX: + ext = "onnx" + onnx_name = self.onnx_name if self.onnx_name is not None else self.name + filepath = f"{models_path}/{self.name}/{self.version}/{onnx_name}.{ext}" + else: + raise NotImplementedError(self.model_type) + return filepath + + def get_bin_path(self): + ext = "bin" + bin_path = None + if self.model_type == ModelType.IR: + bin_path = f"{models_path}/{self.name}/{self.version}/{self.name}.{ext}" + else: + raise NotImplementedError(self.model_type) + return bin_path + + def get_config(self): + config = {} + + for field_name in self.get_list_of_config_fields(): + if field_name == "name": + if self.is_mediapipe: + base_path = getattr(self, "base_path", None) + value = os.path.basename(base_path) + else: + value = getattr(self, field_name, None) + else: + value = getattr(self, field_name, None) + if value is not None: + config[field_name] = value + + if self.target_device is not None: + config["target_device"] = self.target_device + + if self.custom_loader is not None: + config.update(self.custom_loader.model_options) + + config_shape = self._calculate_shape_for_config() + if config_shape: + config["shape"] = config_shape + + config_layout = self._calculate_layout_for_config() + if config_layout: + config["layout"] = config_layout + + if self.is_stateful: + config["stateful"] = True + + return {"config": config} + + def set_input_shape_for_ovms(self, input_shape: Union[str, List, Dict[str, List]] = None): + if input_shape is None: + input_shape = self.input_shapes + + if isinstance(input_shape, dict): + result = {} + for input_name, shape in input_shape.items(): + if isinstance(shape, str): + result[input_name] = shape + elif isinstance(shape, list) or isinstance(shape, tuple): + shape_dims_str = f"{','.join([str(shape_dim) for shape_dim in shape])}" + result[input_name] = f"({shape_dims_str})" + self.input_shape_for_ovms = result + elif isinstance(input_shape, list): + self.input_shape_for_ovms = f"({','.join([str(shape_dim) for shape_dim in input_shape])})" + else: + self.input_shape_for_ovms = input_shape + + return self.input_shape_for_ovms + + def set_layout_for_ovms(self, layout: Union[str, dict]): + if isinstance(layout, str): + self._layout_for_ovms = layout + elif isinstance(layout, dict): + self._layout_for_ovms = json.dumps(layout) + else: + raise NotImplementedError() + + return self._layout_for_ovms + + def try_to_update_batch_size(self, shape): + if shape is not None and isinstance(shape, list) and isinstance(shape[0], int): + self.batch_size = shape[0] + + def _calculate_shape_for_config(self): + return self.input_shape_for_ovms + + def _calculate_layout_for_config(self): + layout_for_config = None + if self._layout_for_ovms is not None: + layout_for_config = self._layout_for_ovms + else: + layouts = [] + + inputs_outputs = {} + if self.inputs is not None: + inputs_outputs.update(self.inputs) + + if self.outputs is not None: + inputs_outputs.update(self.outputs) + + for input_name, input_info in inputs_outputs.items(): + if input_info is not None: + layout = input_info.get("layout", None) + if layout is not None: + layouts.append(f'"{input_name}": "{layout}"') + + if layouts: + layout_for_config = ", ".join(layouts) + else: + layout_for_config = None + + if layout_for_config is not None: + layout_for_config = f"{{{layout_for_config}}}" + + return layout_for_config + + def get_expected_output(self, input_data: dict, client_type: str = None): + return None + + @staticmethod + def is_pipeline(): + return False + + def get_model_path(self): + return self.base_path + + def prepare_input_data(self, batch_size=None, random_data=False, input_key=None): + result = dict() + for input_name, input_data in self.inputs.items(): + if batch_size is not None: + input_data["shape"][0] = batch_size + if input_data["shape"] and input_data["shape"][0] == -1 and input_data.get("dataset", None): + result[input_name] = input_data["dataset"].get_data(input_data["shape"], input_data["shape"][0], False) + else: + if random_data: + result[input_name] = np.random.uniform(-100.0, 100.0, input_data["shape"]).astype( + input_data["dtype"] + ) + else: + result[input_name] = np.ones(input_data["shape"], dtype=input_data["dtype"]) + return result + + def prepare_input_data_from_model_datasets(self, batch_size=None): + result = dict() + for param_name, param_data in self.inputs.items(): + if batch_size is None: + batch_size = self.get_expected_batch_size() if self.batch_size is None else self.batch_size + result[param_name] = param_data["dataset"].get_data( + shape=param_data["shape"], + batch_size=batch_size, + transpose_axes=self.transpose_axes, + datatype=param_data["dtype"], + ) + return result + + def get_model_path_with_version(self, version=None): + if version is None: + version = self.version + return os.path.join(self.get_model_path(), str(version)) + + def get_model_files(self): + model_dir = self.model_path_on_host + all_files = os.listdir(model_dir) + return all_files + + def get_expected_batch_size(self) -> int: + expected_batch_size = None + + def get_batch_size_form_input_shape(): + model = self + if self.inputs is None: + model = self.clone() + if not any([v["shape"] for v in model.inputs.values()]): + return Ovms.SCALAR_BATCH_SIZE + + return [v["shape"] for v in model.inputs.values()][0][0] + + if self.input_shape_for_ovms is not None or self.batch_size is None or self.batch_size == "auto": + expected_batch_size = get_batch_size_form_input_shape() + else: + expected_batch_size = self.batch_size + + try: + if expected_batch_size == Ovms.SCALAR_BATCH_SIZE: + return expected_batch_size + + if isinstance(expected_batch_size, str) and ":" in expected_batch_size: + expected_batch_size = expected_batch_size.split(":")[0] + + expected_batch_size = int(expected_batch_size) + except (TypeError, ValueError) as e: + raise e.__class__(f"Calculated expected_batch_size: `{expected_batch_size}` is not a number: {e}") + + return expected_batch_size + + def clone(self, clone_model_name=None, model_path_on_host=None): + clone = type(self)() + if clone_model_name is not None: + clone.name = clone_model_name + clone.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, clone_model_name) + if model_path_on_host is not None: + clone.model_path_on_host = model_path_on_host + + copy_tree(self.model_path_on_host, clone.model_path_on_host) + return clone + + # REMOTE_SERVER_ADDRESS is not pre-defined, but set "on the fly" in our tests (prepare_remote_k8s_cluster_data), + # based on the remote_ip read from the loaded kubeconfig_file (get_ip_from_kubeconfig_file). + + def create_new_version(self, container_folder, new_version, copy_from_host_path=False, model_name=None): + model_name = model_name if model_name is not None else self.name + result = type(self)() + + if copy_from_host_path: + source = self.model_path_on_host + else: + source = Path(container_folder, Paths.MODELS_PATH_NAME, model_name, str(self.version)) + + destination = Path(container_folder, Paths.MODELS_PATH_NAME, model_name, str(new_version)) + if source != destination: + # This check is for negative tests from TestOnlineModification + shutil.copytree(source, destination, dirs_exist_ok=True) + for file in destination.glob("*"): + # resource files from shared folder should be read only. + # Add proper access for test container folder manipulations. + file.chmod(file.stat().st_mode | stat.S_IWRITE) + + result.version = new_version + + # Copy inputs/outputs (be aware that inputs/outputs can be mapped) + result.inputs = self.inputs + result.outputs = self.outputs + + if result.is_mediapipe: + for model in result.regular_models: + if model_name == model.name: + model.version = new_version + + return result + + def delete(self, container_folder, model_name=None): + model_name = model_name if model_name is not None else self.name + remove_tree(os.path.join(container_folder, Paths.MODELS_PATH_NAME, model_name)) + + def delete_version(self, container_folder): + remove_tree(os.path.join(container_folder, Paths.MODELS_PATH_NAME, self.name, str(self.version))) + + def restore_input_names(self): + model = self.clone() + self.inputs = {} + for input_name in model.inputs: + self.inputs[input_name] = None + + def change_input_name(self, old_name, new_name): + tmp = self.inputs.pop(old_name) + self.inputs[new_name] = tmp + + def change_output_name(self, old_name, new_name): + tmp = self.outputs.pop(old_name) + self.outputs[new_name] = tmp + + def validate_outputs(self, outputs, expected_output_shapes=None, provided_input=None): + assert outputs, "Prediction returned no output" + if expected_output_shapes is None: + expected_output_shapes = list(self.output_shapes.values()) + for i, shape in enumerate(expected_output_shapes): # Check for dynamic shape + for j, val in enumerate(shape): + if val == -1: + expected_output_shapes[i][j] = 1 + + for output_name in self.output_names: + assert ( + output_name in outputs + ), f"Incorrect output name, expected: {output_name}, found: {', '.join(outputs.keys())}" + output_shapes = [list(o.shape) for o in outputs.values()] + assert any( + shape in expected_output_shapes for shape in output_shapes + ), f"Incorrect output shape, expected: {expected_output_shapes}, found: {output_shapes}." + + def get_ovms_loading_time(self): + loading_file_speed = 0.60 if self.is_on_cloud else 300.0 + model_loading_speed = DEVICE_LOADING_SPEED[self.target_device][self.model_type] + if self.custom_loader: + model_loading_speed = model_loading_speed * 0.10 + size = self.size + if size > 0.0: + size_in_mb = math.ceil(size / (1024.0 * 1024.0)) + else: + size_in_mb = 100 # Model files not provided yet assume 100MB model + + timeout = size_in_mb * (1 / loading_file_speed + 1 / model_loading_speed) + timeout += 60 if self.is_on_cloud else 10 # required for very small models on cloud + timeout += 10 if self.custom_loader else 0 # required for additional overhead of custom loader invocation + timeout += 10 if is_nginx_mtls else 0 # required for additional overhead of nginx invocation + + timeout += timeout * 0.2 * xdist_workers + + return timeout + + def prepare_resources(self, base_location): + result = [] + resource_destination = ( + os.path.join(base_location, Paths.MODELS_PATH_NAME) + if not self.use_subconfig + else os.path.join(base_location, Paths.MODELS_PATH_NAME, self.name) + ) + if not self.is_on_cloud and self.model_path_on_host is not None: + if self.copy_all_model_versions: + src_model_path = Path(self.model_path_on_host).parent + target_model_dir = Path(resource_destination, src_model_path.name) + else: + src_model_path = Path(self.model_path_on_host) + # model_name/version_num + model_subpath = src_model_path.parts[-2:] if self.model_subpath is None else Path(self.model_subpath).parts + target_model_dir = Path(resource_destination, *model_subpath) + if not os.path.exists(target_model_dir): + logger.debug(f"Copying {self.name} to container: {target_model_dir}") + shutil.copytree(src_model_path, target_model_dir, dirs_exist_ok=True) + for file in target_model_dir.glob("*"): + # resource files from shared folder should be read only. + # Add proper access for test container folder manipulations. + file.chmod(file.stat().st_mode | stat.S_IWRITE) + result = [resource_destination] + else: + if CLOUD_HEADERS["google"] in self.base_path: + # WA for GoogleCloud credential folder + result = [resource_destination, os.path.join(TestEnvironment.current.base_dir, "credentials")] + + if self.custom_loader: + assert not self.is_on_cloud, "Test framework not ready for models on cloud with custom_loader!" + if self.custom_loader.prepare_custom_loader_resources: + result.append(self.custom_loader.prepare_resources(base_location)) + return result + + def get_input_shape(self, input_name): + return Shape(self.inputs[input_name]["shape"], self._compiled_layout) + + def set_shape_for_input(self, input_name, shape): + _layout = self._compiled_layout.split(":")[0] if self._compiled_layout else None + self.inputs[input_name]["shape"] = shape.get_shape_by_layout(_layout) + + def change_input_layout(self, new_layout): + new_layout = new_layout.split(":")[0] if ":" in new_layout else new_layout + for input_name, val in self.inputs.items(): + s = val["shape"] + new_shape = [s[0], s[2], s[3], s[1]] + val["shape"] = new_shape + + def change_input_type(self, input, type): + self.inputs[input]["dtype"] = type + + def get_regular_models(self): + return [self] + + def get_demultiply_count(self): + return None + + def get_mapping_config_path(self, container_folder): + return OvmsMappingConfig.mapping_config_path(container_folder, self) + + def get_mapping_dict(self, container_folder): + mapping_config_path = self.get_mapping_config_path(container_folder) + return OvmsMappingConfig.load_config(mapping_config_path) + + @property + def is_on_cloud(self): + result = False + for header in CLOUD_HEADERS.values(): + if header in self.base_path: + result = True + break + return result + + @property + def size(self): + if self.model_path_on_host is None or not os.path.exists(self.model_path_on_host): + return 0.0 + file_list = os.listdir(self.model_path_on_host) + file_ext = ".bin" if self.model_type == ModelType.IR else ".onnx" + detected = [x for x in file_list if file_ext in x] + result = 1 + if len(detected) > 0: + result = Path(self.model_path_on_host, detected[0]).stat().st_size + + return result + + @property + def input_names(self): + return list(self.inputs.keys()) + + @property + def output_names(self): + return list(self.outputs.keys()) + + @property + def input_shapes(self): + return {k: v["shape"] for k, v in self.inputs.items()} + + @input_shapes.setter + def input_shapes(self, shape): + for k, v in self.inputs.items(): + v["shape"] = shape + + @property + def input_layouts(self): + return {k: v.get("layout", None) for k, v in self.inputs.items()} if self.inputs else {} + + @input_layouts.setter + def input_layouts(self, layout): + for k, v in self.inputs.items(): + v["layout"] = layout + + @property + def output_shapes(self): + return {k: v["shape"] for k, v in self.outputs.items()} + + @property + def input_types(self): + return {k: v["dtype"] for k, v in self.inputs.items()} + + @property + def output_types(self): + return {k: v["dtype"] for k, v in self.outputs.items()} + + @property + def input_datasets(self): + return {k: v["dataset"] if "dataset" in v else None for k, v in self.inputs.items()} + + def is_dynamic(self): + return False + + @staticmethod + def rename_input_ouput_data(data, src_name, dst_name): + data[dst_name] = data[src_name] + del data[src_name] + return data diff --git a/tests/functional/models/models_datasets.py b/tests/functional/models/models_datasets.py new file mode 100644 index 0000000000..54edec2530 --- /dev/null +++ b/tests/functional/models/models_datasets.py @@ -0,0 +1,261 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import base64 +import json +import os +import re +from dataclasses import dataclass, field +from io import BytesIO +from random import choice +from string import ascii_lowercase + +import cv2 +import numpy as np +from PIL import Image + +from tests.functional.config import binary_io_images_path, datasets_path +from tests.functional.constants.ovms import Ovms +from tests.functional.utils.inference.serving.openai import ChatCompletionsApi +from tests.functional.utils.numpy_loader import prepare_data + + +class ModelDataset: + + @staticmethod + def create(data_str): + result = None + ext = os.path.splitext(data_str)[1] + if ext == ".npy": + result = NumPyDataset(data_str) + return result + + def __init__(self): + self.data_path = None + self.name = None + self.shape = None + + def get_data(self, shape, batch_size, transpose_axes, layout=None, datatype=np.float32): + data = prepare_data( + data_path=self.data_path, + expected_shape=shape, + batch_size=batch_size, + transpose_axes=transpose_axes, + expected_layout=layout, + data_layout=None, + ) + self.shape = data.shape + return data + + def to_str(self): + return json.dumps(self.__dict__) + + +class NumPyDataset(ModelDataset): + + def __init__(self, *data_path): + self.name = data_path[0] + self.data_path = os.path.join(datasets_path, *data_path) + + +class LanguageModelDataset(ModelDataset): + str_input_data = ["Lorem ipsum dolor sit amet", "consectetur adipiscing elit", "sed do eiusmod tempor"] + + def __init__(self, data_sample=0): + try: + self.default_str_input_data = self.str_input_data[data_sample] + except IndexError: + self.default_str_input_data = self.str_input_data[-1] + + def get_data(self, shape, batch_size, transpose_axes, layout=None, datatype=np.float32): + # https://github.com/triton-inference-server/client/blob/main/src/python/examples/simple_grpc_string_infer_client.py + str_input_data = [""] if batch_size == 0 else [s.split() for s in self.get_str_input_data()] + return np.array([str(x).encode("utf-8") for x in str_input_data], dtype=np.object_) + + def get_str_input_data(self): + return [self.default_str_input_data] + + def get_source_data(self, shape, dtype=np.float32): + return {} + + def create_data(self, tmp_file_location, shape, img_format): + return {} + + @staticmethod + def generate_random_text_list(inputs_number, word_length=5): + return ["".join(choice(ascii_lowercase) for _ in range(word_length)) for _ in range(inputs_number)] + + +class LargeLanguageModelDataset(ModelDataset): + user_content = "What is OpenVINO?" + system_content = "You are a helpful assistant." + user_data = [ChatCompletionsApi.ROLE_USER, user_content] + system_data = [ChatCompletionsApi.ROLE_SYSTEM, system_content] + input_data = [system_data, user_data] + + def __init__(self, data_sample=0): + self.default_input_data = self.input_data + + def get_data(self, shape, batch_size, transpose_axes, layout=None, datatype=np.float32): + return self.default_input_data + + def get_source_data(self, shape, dtype=np.float32): + return {} + + def create_data(self, tmp_file_location, shape, img_format): + return {} + + +class FeatureExtractionModelDataset(LargeLanguageModelDataset): + input_data_1 = "That is a happy person." + input_data_2 = "That is a very happy person." + input_data = [input_data_1, input_data_2] + + def __init__(self, data_sample=0): + self.default_input_data = self.input_data + + def get_string_data(self): + return self.input_data_1 + + +class RerankModelDataset(LargeLanguageModelDataset): + query = "hello" + document_1 = "welcome" + document_2 = "farewell" + input_data = { + "query": query, + "documents": [document_1, document_2] + } + + def __init__(self, data_sample=0): + self.default_input_data = self.input_data + + def get_string_data(self): + return str(self.input_data) + + +class SingleMessageLanguageModelDataset(LargeLanguageModelDataset): + user_data = [ChatCompletionsApi.ROLE_USER, LargeLanguageModelDataset.user_content] + input_data = [user_data] + + +def load_image_data_from_path(full_path, img_format, img_mode=None, size=None): + img_byte_arr = BytesIO() + image_obj = Image.open(full_path, mode="r", formats=None) + if img_mode: + image_obj = image_obj.convert(img_mode) + if size: + image_obj = image_obj.resize(size) + image_obj.save(img_byte_arr, format=img_format) + return img_byte_arr.getvalue() + + +class BinaryDummyModelDataset(ModelDataset): + def get_data(self, shape, batch_size, transpose_axes, layout=None, datatype=np.float32): + return np.ones(shape, datatype) + + def get_source_data(self, shape, dtype=np.float32): + return np.ones(shape, dtype) + + def create_data(self, tmp_file_location, shape, img_format): + data = self.get_source_data(shape) + fname = f'generated_ones_{"x".join([str(x) for x in shape])}.{img_format.lower()}' + os.makedirs(tmp_file_location, exist_ok=True) + cv2.imwrite(os.path.join(tmp_file_location, fname), data) + return load_image_data_from_path(os.path.join(tmp_file_location, fname), img_format) + + +@dataclass +class DefaultBinaryDataset(ModelDataset): + _saved_labels_to_path_mapping: str = None + image_format: str = None + image_mode: str = None + max_num_of_images: int = None + offset: int = 0 + + def get_path(self): + return os.path.join(binary_io_images_path, "input_images.txt") + + def _get_image_label_mapping(self): + image_list_path = self.get_path() + image_labels = {} + with open(image_list_path, "r") as f: + for line in f.readlines(): + path, label = line.strip().split(" ") + image_labels[path] = label + return image_labels + + def get_data(self, shape, batch_size, transpose_axes, layout, reshape=False, datatype=np.float32): + labels_to_path_mapping = self._get_image_label_mapping() + + i = 0 + images = [] + size = shape[-2:] if reshape else None + for path, label in labels_to_path_mapping.items(): + i += 1 + if i <= self.offset: + continue + + full_path = os.path.join(binary_io_images_path, path) + img_data = load_image_data_from_path(full_path, self.image_format, size=size) + images.append(img_data) + + if len(images) == batch_size: + break + + self._saved_labels_to_path_mapping = labels_to_path_mapping + return images + + def verify_match(self, response): + result = True + + if self._saved_labels_to_path_mapping is None: + return result + + labels = list(self._saved_labels_to_path_mapping.values()) + + nu = list(response.items()) + assert len(nu) == 1 # We expect single dimension result + nu = nu[0][1] + + if nu.shape[-1] == 1001: + model_offset = 1 + else: + model_offset = 0 + + for i in range(nu.shape[0]): + label = int(labels[i + self.offset]) + single_result = nu[[i], ...] + ma = np.argmax(single_result) - model_offset + if label != ma: + result = False + return result + + +@dataclass +class ExactShapeBinaryDataset(DefaultBinaryDataset): + shape: dict = field(default_factory=lambda: []) + image_format: str = Ovms.JPG_IMAGE_FORMAT + + def get_path(self): + file_path = os.path.join(binary_io_images_path, "images", "_".join([str(x) for x in self.shape])) + file_names = list(filter(lambda x: re.match(r".+\.jpe?g$", x.lower()), os.listdir(file_path))) + assert len(file_names) == 1, f"Unable to find images file with shape: {self.shape}" + return os.path.join(file_path, file_names[0]) + + def get_data(self, shape, batch_size, transpose_axes, layout=None, reshape=False, datatype=np.float32): + path = self.get_path() + return [load_image_data_from_path(path, self.image_format)] diff --git a/tests/functional/models/models_generative.py b/tests/functional/models/models_generative.py new file mode 100644 index 0000000000..b79b2d99f6 --- /dev/null +++ b/tests/functional/models/models_generative.py @@ -0,0 +1,186 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import shutil + +from dataclasses import dataclass +from pathlib import Path + +from tests.functional.config import generative_models_local_path +from tests.functional.constants.paths import Paths +from tests.functional.models.models import ModelInfo, ModelType +from tests.functional.models.models_datasets import ( + FeatureExtractionModelDataset, + LargeLanguageModelDataset, + RerankModelDataset, + SingleMessageLanguageModelDataset, +) + + +@dataclass +class GenerativeModel(ModelInfo): + model_type: ModelType = ModelType.IR + is_generative: bool = True + is_local: bool = True + precision: str = "INT8" + precision_dir: str = "INT8" + parent_name: str = None + parent_base_dir: str = os.path.join("pytorch", "ov") + parent_precision_dir: str = "OV_FP16-INT8_ASYM" + max_position_embeddings: int = None + model_path_on_parent_host: str = None + model_subpath: str = None + single_message_dataset: bool = False + allows_reasoning: bool = False + is_llm: bool = False + is_feature_extraction: bool = False + is_rerank: bool = False + is_image_generation: bool = False + is_audio: bool = False + is_hf_direct_load: bool = False + is_agentic: bool = False + gguf_filename: str = None + pooling: str = None + transformers_v4_required: bool = False + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls._own_field_defaults = {} + for name in getattr(cls, '__annotations__', {}): + if name in cls.__dict__: + cls._own_field_defaults[name] = cls.__dict__[name] + + def __post_init__(self): + self.model_base_path_on_host = generative_models_local_path + self.model_subpath = os.path.join(self.precision_dir, Path(self.name)) + self.model_path_on_host = os.path.join(self.model_base_path_on_host, self.model_subpath) + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, self.model_subpath) + + def get_default_dataset(self): + return + + def prepare_input_data(self, batch_size=None, input_key=None, dataset=None, input_data_type=None): + if dataset is not None: + input_data = {input_name: dataset().get_data(None, None, None) for input_name in self.input_names} + elif input_data_type == "string": + input_data = { + input_name: self.inputs[input_name]["dataset"].get_string_data() + for input_name in self.input_names + } + else: + input_data = { + input_name: self.inputs[input_name]["dataset"].get_data(None, None, None) + for input_name in self.input_names + } + return input_data + + +@dataclass +class LargeLanguageModel(GenerativeModel): + is_llm: bool = True + + def get_default_dataset(self): + if self.single_message_dataset: + return SingleMessageLanguageModelDataset + return LargeLanguageModelDataset + + +@dataclass +class FeatureExtractionModel(GenerativeModel): + use_subconfig: bool = True + is_feature_extraction: bool = True + pooling: str = "CLS" + + def get_default_dataset(self): + return FeatureExtractionModelDataset + + +@dataclass +class RerankModel(GenerativeModel): + is_rerank: bool = True + use_subconfig: bool = True + + @staticmethod + def get_default_dataset(): + return RerankModelDataset + + +@dataclass +class GenerativeModelHuggingFace(GenerativeModel): + is_local: bool = False + is_hf_direct_load: bool = True + input_name: str = "input" + precision: str = "INT4" + model_timeout: int = 900 + + def _apply_diamond_defaults(self): + """Fix field defaults for diamond inheritance. + + When a class inherits from both LargeLanguageModelHuggingFace and a specialized + type (e.g. ImageGenerationModel), LargeLanguageModelHuggingFace's inherited field + defaults override the specialized type's directly-defined defaults. This method + restores the correct defaults from specialized parent classes. + """ + cls = type(self) + seen_fields = set() + seen_fields.update(getattr(cls, '_own_field_defaults', {}).keys()) + seen_fields.update(getattr(GenerativeModelHuggingFace, '_own_field_defaults', {}).keys()) + for base in cls.__mro__: + if base in (cls, object, GenerativeModelHuggingFace, GenerativeModel, ModelInfo): + continue + own_defaults = getattr(base, '_own_field_defaults', {}) + for field_name, default_value in own_defaults.items(): + if field_name not in seen_fields: + setattr(self, field_name, default_value) + seen_fields.add(field_name) + + def __post_init__(self): + self._apply_diamond_defaults() + if self.is_local: + self.model_base_path_on_host = generative_models_local_path + self.model_path_on_host = os.path.join(self.model_base_path_on_host, Path(self.name)) + self.model_subpath = os.path.join("models_ov_hf", Path(self.name)) + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, self.name) + self.set_additional_model_params() + + def prepare_resources(self, base_location): + models_dir = Path(base_location, Paths.MODELS_PATH_NAME) + models_dir.mkdir(exist_ok=True, parents=True) + if self.is_local: + models_sub_dir = Path(models_dir, self.name) + if not models_sub_dir.exists(): + shutil.copytree(self.model_path_on_host, models_sub_dir) + return [str(models_dir)] + + def prepare_input_data(self, batch_size=None, input_key=None, dataset=None, input_data_type=None): + if dataset is not None: + dataset_obj = dataset if not isinstance(dataset, type) else dataset() + else: + dataset_obj = self.get_default_dataset()() + if input_data_type == "string": + input_data = {self.input_name: dataset_obj.get_string_data()} + else: + input_data = {self.input_name: dataset_obj.get_data(None, None, None)} + return input_data + + +@dataclass +class Qwen3Embedding06BFp16OvHf(GenerativeModelHuggingFace, FeatureExtractionModel): + name: str = "OpenVINO/Qwen3-Embedding-0.6B-fp16-ov" + precision: str = "FP16" + pooling: str = "LAST" + is_local: bool = True diff --git a/tests/functional/models/models_library.py b/tests/functional/models/models_library.py new file mode 100644 index 0000000000..982dccd4c8 --- /dev/null +++ b/tests/functional/models/models_library.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.models.models_generative import Qwen3Embedding06BFp16OvHf + + +class ModelsLibrary: + + @property + def various_feature_extraction_models(self): + return [Qwen3Embedding06BFp16OvHf] + + +ModelsLib = ModelsLibrary() From 403b0f374dde2ec66fa79117cb99f0939365b48b Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Fri, 19 Jun 2026 18:32:40 +0200 Subject: [PATCH 08/13] 19/06/2026 part 4 --- tests/functional/config.py | 9 +- tests/functional/conftest.py | 12 +- tests/functional/models/models.py | 32 +- tests/functional/models/models_datasets.py | 16 +- tests/functional/models/models_generative.py | 8 +- tests/functional/models/models_library.py | 2 +- tests/functional/pylintrc | 749 ++++++++++++++++++ tests/functional/test_embeddings.py | 2 + tests/functional/utils/assertions.py | 4 + tests/functional/utils/download.py | 4 +- tests/functional/utils/generative_ai/utils.py | 3 + .../utils/generative_ai/validation_utils.py | 1 + tests/functional/utils/ov_hf_downloader.py | 3 +- tests/functional/utils/test_framework.py | 3 + tests/requirements.txt | 1 + 15 files changed, 812 insertions(+), 37 deletions(-) create mode 100644 tests/functional/pylintrc diff --git a/tests/functional/config.py b/tests/functional/config.py index 1b02977d3c..cd7f2d5f67 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -65,10 +65,6 @@ def get_uses_mapping(): """TEST_DIR_CACHE - location where models and test data should be downloaded to and serve as cache for TEST_DIR""" test_dir_cache = os.environ.get("TEST_DIR_CACHE", "/tmp/ovms_models_cache") -"""TEST_DIR_CLEANUP - if set to True, TEST_DIR directory will be removed after tests execution""" -test_dir_cleanup = os.environ.get("TEST_DIR_CLEANUP", "True") -test_dir_cleanup = test_dir_cleanup.lower() == "true" - """ TT_OVMS_C_REPO_PATH - path to ovms-c repository. Can be relative or absolute. """ ovms_c_repo_path = get_path("TT_OVMS_C_REPO_PATH", get_path("PWD", "./")) @@ -328,6 +324,11 @@ def get_uses_mapping(): __base_os = os.environ.get("BASE_OS", OsType.Ubuntu24) base_os = get_list("TT_BASE_OS", fallback=[__base_os]) +"""" BASE_IMAGE - Docker image used during OVMS image creation""" +base_image = os.environ.get("BASE_IMAGE", None) +if base_image is not None: + assert len(base_os) == 1, "If you wish to iterate by TT_BASE_OS: do not set BASE_IMAGE explicitly." + """ GLOBAL_TEMP_DIR - global temporary directory """ global_tmp_dir_default = os.path.join("~", "AppData", "Local", "Temp") if OsType.Windows in base_os else "/tmp" global_tmp_dir = get_path("GLOBAL_TEMP_DIR", global_tmp_dir_default) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index b0626ef04c..0844f999e0 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -54,9 +54,9 @@ if enable_pytest_plugins: - raise NotImplementedError("OVMS tests not enabled") + # raise NotImplementedError("OVMS tests not enabled") - pytest_plugins = [ + pytest_plugins = [ # pylint: disable=unreachable "tests.functional.fixtures.ovms", "tests.functional.fixtures.server", "tests.functional.fixtures.api_type", @@ -119,7 +119,7 @@ def pytest_unconfigure(config): hooks.teardown_environment() if machine_is_reserved_for_test_session: hooks.clear_lockfiles() - except Exception as e: + except Exception as e: # pylint: disable=broad-exception-caught error_msg = str(e) print(error_msg) sys.exit(error_msg) @@ -134,7 +134,7 @@ def pytest_configure_node(node): def pytest_sessionstart(session): - logger.info("Starting test session in the following folder: {}".format(session.startdir)) + logger.info(f"Starting test session in the following folder: {session.startdir}") log_configuration_variables() session.start_time = time.time() @@ -146,7 +146,7 @@ def pytest_collection_modifyitems(session, config, items): Support for running tests with component tags. Report all test component markers to mongo_reporter. """ - logger.info("Preparing tests for test session in the following folder: {}".format(session.startdir)) + logger.info(f"Preparing tests for test session in the following folder: {session.startdir}") if pytest_keyword_filter: # Filter case insensitive @@ -170,7 +170,7 @@ def pytest_collection_modifyitems(session, config, items): # https://docs.pytest.org/en/6.2.x/reference.html#id58 @pytest.hookimpl(hookwrapper=True, tryfirst=True) -def pytest_runtest_protocol(item: "Item", nextitem: "Optional[Item]"): +def pytest_runtest_protocol(item: "Item"): """ Perform the runtest protocol for a single test item. The default runtest protocol is this (see individual hooks for full details): diff --git a/tests/functional/models/models.py b/tests/functional/models/models.py index 09f1c17ec0..d602a7929c 100644 --- a/tests/functional/models/models.py +++ b/tests/functional/models/models.py @@ -14,13 +14,16 @@ # limitations under the License. # +# pylint: disable=too-many-instance-attributes +# pylint: disable=too-many-public-methods +# pylint: disable=unused-argument + import json import math import os import shutil import stat from dataclasses import dataclass -from distutils.dir_util import copy_tree, remove_tree from enum import Enum from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -228,7 +231,7 @@ def set_input_shape_for_ovms(self, input_shape: Union[str, List, Dict[str, List] for input_name, shape in input_shape.items(): if isinstance(shape, str): result[input_name] = shape - elif isinstance(shape, list) or isinstance(shape, tuple): + elif isinstance(shape, (list, tuple)): shape_dims_str = f"{','.join([str(shape_dim) for shape_dim in shape])}" result[input_name] = f"({shape_dims_str})" self.input_shape_for_ovms = result @@ -297,7 +300,7 @@ def get_model_path(self): return self.base_path def prepare_input_data(self, batch_size=None, random_data=False, input_key=None): - result = dict() + result = {} for input_name, input_data in self.inputs.items(): if batch_size is not None: input_data["shape"][0] = batch_size @@ -313,7 +316,7 @@ def prepare_input_data(self, batch_size=None, random_data=False, input_key=None) return result def prepare_input_data_from_model_datasets(self, batch_size=None): - result = dict() + result = {} for param_name, param_data in self.inputs.items(): if batch_size is None: batch_size = self.get_expected_batch_size() if self.batch_size is None else self.batch_size @@ -342,7 +345,7 @@ def get_batch_size_form_input_shape(): model = self if self.inputs is None: model = self.clone() - if not any([v["shape"] for v in model.inputs.values()]): + if not any(v["shape"] for v in model.inputs.values()): return Ovms.SCALAR_BATCH_SIZE return [v["shape"] for v in model.inputs.values()][0][0] @@ -373,7 +376,7 @@ def clone(self, clone_model_name=None, model_path_on_host=None): if model_path_on_host is not None: clone.model_path_on_host = model_path_on_host - copy_tree(self.model_path_on_host, clone.model_path_on_host) + shutil.copytree(self.model_path_on_host, clone.model_path_on_host, dirs_exist_ok=True) return clone # REMOTE_SERVER_ADDRESS is not pre-defined, but set "on the fly" in our tests (prepare_remote_k8s_cluster_data), @@ -412,10 +415,10 @@ def create_new_version(self, container_folder, new_version, copy_from_host_path= def delete(self, container_folder, model_name=None): model_name = model_name if model_name is not None else self.name - remove_tree(os.path.join(container_folder, Paths.MODELS_PATH_NAME, model_name)) + shutil.rmtree(os.path.join(container_folder, Paths.MODELS_PATH_NAME, model_name)) def delete_version(self, container_folder): - remove_tree(os.path.join(container_folder, Paths.MODELS_PATH_NAME, self.name, str(self.version))) + shutil.rmtree(os.path.join(container_folder, Paths.MODELS_PATH_NAME, self.name, str(self.version))) def restore_input_names(self): model = self.clone() @@ -483,7 +486,8 @@ def prepare_resources(self, base_location): else: src_model_path = Path(self.model_path_on_host) # model_name/version_num - model_subpath = src_model_path.parts[-2:] if self.model_subpath is None else Path(self.model_subpath).parts + model_subpath = src_model_path.parts[-2:] if self.model_subpath is None else \ + Path(self.model_subpath).parts target_model_dir = Path(resource_destination, *model_subpath) if not os.path.exists(target_model_dir): logger.debug(f"Copying {self.name} to container: {target_model_dir}") @@ -513,13 +517,13 @@ def set_shape_for_input(self, input_name, shape): def change_input_layout(self, new_layout): new_layout = new_layout.split(":")[0] if ":" in new_layout else new_layout - for input_name, val in self.inputs.items(): + for _, val in self.inputs.items(): s = val["shape"] new_shape = [s[0], s[2], s[3], s[1]] val["shape"] = new_shape - def change_input_type(self, input, type): - self.inputs[input]["dtype"] = type + def change_input_type(self, input_name, dtype): + self.inputs[input_name]["dtype"] = dtype def get_regular_models(self): return [self] @@ -570,7 +574,7 @@ def input_shapes(self): @input_shapes.setter def input_shapes(self, shape): - for k, v in self.inputs.items(): + for _, v in self.inputs.items(): v["shape"] = shape @property @@ -579,7 +583,7 @@ def input_layouts(self): @input_layouts.setter def input_layouts(self, layout): - for k, v in self.inputs.items(): + for _, v in self.inputs.items(): v["layout"] = layout @property diff --git a/tests/functional/models/models_datasets.py b/tests/functional/models/models_datasets.py index 54edec2530..26e1bdc428 100644 --- a/tests/functional/models/models_datasets.py +++ b/tests/functional/models/models_datasets.py @@ -1,4 +1,4 @@ -# + # Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +14,11 @@ # limitations under the License. # -import base64 +# pylint: disable=arguments-renamed +# pylint: disable=super-init-not-called +# pylint: disable=too-many-positional-arguments +# pylint: disable=unused-argument + import json import os import re @@ -174,13 +178,13 @@ def create_data(self, tmp_file_location, shape, img_format): data = self.get_source_data(shape) fname = f'generated_ones_{"x".join([str(x) for x in shape])}.{img_format.lower()}' os.makedirs(tmp_file_location, exist_ok=True) - cv2.imwrite(os.path.join(tmp_file_location, fname), data) + cv2.imwrite(os.path.join(tmp_file_location, fname), data) # pylint: disable=no-member return load_image_data_from_path(os.path.join(tmp_file_location, fname), img_format) @dataclass class DefaultBinaryDataset(ModelDataset): - _saved_labels_to_path_mapping: str = None + _saved_labels_to_path_mapping: dict = None image_format: str = None image_mode: str = None max_num_of_images: int = None @@ -192,7 +196,7 @@ def get_path(self): def _get_image_label_mapping(self): image_list_path = self.get_path() image_labels = {} - with open(image_list_path, "r") as f: + with open(image_list_path, "r", encoding="utf-8") as f: for line in f.readlines(): path, label = line.strip().split(" ") image_labels[path] = label @@ -204,7 +208,7 @@ def get_data(self, shape, batch_size, transpose_axes, layout, reshape=False, dat i = 0 images = [] size = shape[-2:] if reshape else None - for path, label in labels_to_path_mapping.items(): + for path, _ in labels_to_path_mapping.items(): i += 1 if i <= self.offset: continue diff --git a/tests/functional/models/models_generative.py b/tests/functional/models/models_generative.py index b79b2d99f6..4ea438283c 100644 --- a/tests/functional/models/models_generative.py +++ b/tests/functional/models/models_generative.py @@ -14,6 +14,9 @@ # limitations under the License. # +# pylint: disable=arguments-renamed +# pylint: disable=too-many-instance-attributes + import os import shutil @@ -114,8 +117,7 @@ class RerankModel(GenerativeModel): is_rerank: bool = True use_subconfig: bool = True - @staticmethod - def get_default_dataset(): + def get_default_dataset(self): return RerankModelDataset @@ -170,7 +172,7 @@ def prepare_input_data(self, batch_size=None, input_key=None, dataset=None, inpu if dataset is not None: dataset_obj = dataset if not isinstance(dataset, type) else dataset() else: - dataset_obj = self.get_default_dataset()() + dataset_obj = self.get_default_dataset()() # pylint: disable=not-callable if input_data_type == "string": input_data = {self.input_name: dataset_obj.get_string_data()} else: diff --git a/tests/functional/models/models_library.py b/tests/functional/models/models_library.py index 982dccd4c8..a095ad7caf 100644 --- a/tests/functional/models/models_library.py +++ b/tests/functional/models/models_library.py @@ -24,4 +24,4 @@ def various_feature_extraction_models(self): return [Qwen3Embedding06BFp16OvHf] -ModelsLib = ModelsLibrary() +ModelsLib = ModelsLibrary() # pylint: disable=invalid-name diff --git a/tests/functional/pylintrc b/tests/functional/pylintrc new file mode 100644 index 0000000000..346b484f5c --- /dev/null +++ b/tests/functional/pylintrc @@ -0,0 +1,749 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +# ignore= + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + config.py, + constants/metrics.py, + constants/ovms.py, + constants/ovms_images.py, + constants/ovms_messages.py, # W0511: + constants/ovms_openai.py, + constants/paths.py, + constants/pipelines.py, # (R0801: duplicate code with models.py) + constants/target_device_configuration.py, # W0105: String statement has no effect (pointless-string-statement) + data/ovms_capi_wrapper/ovms_autopxd.py, + data/ovms_capi_wrapper/setup.py, + data/python_custom_nodes/incrementer/incrementer.py, + data/python_custom_nodes/ovms_basic/python_model.py, + data/python_custom_nodes/ovms_basic/python_model_loopback.py, + data/python_custom_nodes/ovms_corrupted/python_model_corrupted_import.py, + data/python_custom_nodes/ovms_corrupted/python_model_exceptions.py, + data/python_custom_nodes/ovms_corrupted/python_model_loopback_multiple_use_of_valid_outputs.py, + data/python_custom_nodes/ovms_corrupted/python_model_loopback_return_instead_of_yield.py, + data/python_custom_nodes/ovms_corrupted/python_model_writing_to_loopback_output_in_execute.py, + fixtures/*, + object_model/cpu_extension.py, # C0103: Attribute name doesn't conform to naming style (invalid-name) + object_model/custom_loader.py, + object_model/custom_node.py, + object_model/dmesg_log_monitor.py, + object_model/inference_helpers.py, + object_model/mediapipe_calculators.py, + object_model/ovms_binary.py, + object_model/ovms_capi.py, + object_model/ovms_command.py, + object_model/ovms_config.py, # (R0801: duplicate code with ovms_mapping_config.py) + object_model/ovms_docker.py, + object_model/ovms_info.py, + object_model/ovms_instance.py, + object_model/ovms_log_monitor.py, + object_model/ovms_mapping_config.py, # (R0801: duplicate code with ovms_config.py) + object_model/ovms_params.py, + object_model/ovsa.py, + object_model/package_manager.py, + object_model/python_custom_nodes/python_custom_nodes.py, + object_model/resource_monitor.py, + object_model/shape.py, + object_model/test_environment.py, # R0205: (useless-object-inheritance) + object_model/test_helpers.py, + utils/assertions.py, + utils/context.py, + utils/core.py, + utils/docker.py, + utils/git_operations.py, + utils/hooks.py, + utils/http/base.py, + utils/http/client_auth/auth.py, + utils/http/client_auth/base.py, + utils/http/http_client.py, + utils/http/http_client_configuration.py, + utils/http/http_client_factory.py, + utils/http/http_session.py, + utils/http/http_socket_wrapper.py, + utils/inference/capi.py, + utils/inference/communication/base.py, + utils/inference/communication/grpc.py, + utils/inference/communication/rest.py, + utils/inference/inference_client_factory.py, + utils/inference/serving/base.py, + utils/inference/serving/kf.py, + utils/inference/serving/openai.py, + utils/inference/serving/tf.py, + utils/inference/serving/triton.py, + utils/log_monitor.py, + utils/logger.py, + utils/marks.py, + utils/numpy_loader.py, + utils/port_manager.py, + utils/process.py, + utils/reservation_manager/__init__.py, + utils/reservation_manager/__main__.py, + utils/reservation_manager/args.py, + utils/reservation_manager/locker.py, + utils/reservation_manager/manager.py, + utils/reservation_manager/manager_config.py, + utils/reservation_manager/runner.py, + utils/reservation_manager/unittests/test_manager.py, + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=openai + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +init-hook='import sys; sys.path.insert(0, ".")' + + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.10 + +# Discover python modules and packages in the file system subtree. +recursive=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names= + foo, + bar, + baz, + toto, + tutu, + tata, + dupa, + kaczka + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names= + i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods= + __init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected= + _asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=8 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions= + builtins.BaseException, + builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence= + HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable= + raw-checker-failed, # (I0001) + bad-inline-option, # (I0010) + locally-disabled, # (I0011) + file-ignored, # (I0013) + suppressed-message, # (I0020) + useless-suppression, # (I0021) + deprecated-pragma, # (I0022) + use-symbolic-message-instead, # (I0023) + logging-fstring-interpolation, # (W1203, to be FIXED) + logging-format-interpolation, # (W1202, to be FIXED) + missing-function-docstring, # (C0116, to be FIXED) + missing-class-docstring, # (C0115, to be FIXED) + missing-module-docstring, # (C0114, to be FIXED) + too-few-public-methods, # (R0903, to be FIXED) + too-many-lines, # (allowed intentionally) + too-many-statements, # (allowed intentionally) + too-many-locals, # (allowed intentionally) + too-many-branches, # (allowed intentionally) + + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods= + requests.api.delete, + requests.api.get, + requests.api.head, + requests.api.options, + requests.api.patch, + requests.api.post, + requests.api.put, + requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions= + sys.exit, + argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation= + max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=colorized + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives= + fmt: on, + fmt: off, + noqa:, + noqa, + nosec, + isort:skip, + mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=random.getrandbits + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins= + no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes= + optparse.Values, + thread._local, + _thread._local, + argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks= + cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules= + six.moves, + past.builtins, + future.builtins, + builtins,io diff --git a/tests/functional/test_embeddings.py b/tests/functional/test_embeddings.py index ae26ac515c..9c43c4bb53 100644 --- a/tests/functional/test_embeddings.py +++ b/tests/functional/test_embeddings.py @@ -14,6 +14,8 @@ # limitations under the License. # +# pylint: disable=too-many-positional-arguments + import pytest from tests.functional.models.models_library import ModelsLib diff --git a/tests/functional/utils/assertions.py b/tests/functional/utils/assertions.py index 9bbb83faef..b0023a55d4 100644 --- a/tests/functional/utils/assertions.py +++ b/tests/functional/utils/assertions.py @@ -385,6 +385,10 @@ class OVVPException(OvmsTestException): pass +class OVHfDownloadException(OvmsTestException): + pass + + def get_exception_by_ovms_log(ovms_log_lines): exceptions_to_recognize = [NginxException] diff --git a/tests/functional/utils/download.py b/tests/functional/utils/download.py index 17f40d39ec..8f589a9acc 100644 --- a/tests/functional/utils/download.py +++ b/tests/functional/utils/download.py @@ -29,7 +29,7 @@ def wget_item(dst, cmd): proc.set_log_silence() proc.policy["log-check-output"]["stderr"] = False - with SelfDeletingFileLock(f"{dst}.lock", self_delete=True) as fl: + with SelfDeletingFileLock(f"{dst}.lock", self_delete=True) as _: proc.run_and_check(cmd) @@ -45,7 +45,7 @@ def curl_file(url, dst, user, token): proc = Process() proc.set_log_silence() proc.policy["log-check-output"]["stderr"] = False - with SelfDeletingFileLock(f"{dst}.lock", self_delete=True) as fl: + with SelfDeletingFileLock(f"{dst}.lock", self_delete=True) as _: proc.run_and_check(cmd) diff --git a/tests/functional/utils/generative_ai/utils.py b/tests/functional/utils/generative_ai/utils.py index c9bd1e0c56..83245004a9 100644 --- a/tests/functional/utils/generative_ai/utils.py +++ b/tests/functional/utils/generative_ai/utils.py @@ -14,6 +14,9 @@ # limitations under the License. # +# pylint: disable=too-many-arguments +# pylint: disable=too-many-positional-arguments + from openai import NotFoundError from tests.functional.config import ( diff --git a/tests/functional/utils/generative_ai/validation_utils.py b/tests/functional/utils/generative_ai/validation_utils.py index 2ce038123d..7bb4b6b92b 100644 --- a/tests/functional/utils/generative_ai/validation_utils.py +++ b/tests/functional/utils/generative_ai/validation_utils.py @@ -15,6 +15,7 @@ # # pylint: disable=too-many-nested-blocks +# pylint: disable=too-many-positional-arguments # pylint: disable=unused-argument import base64 diff --git a/tests/functional/utils/ov_hf_downloader.py b/tests/functional/utils/ov_hf_downloader.py index 55cd9f5750..cc2616e21d 100644 --- a/tests/functional/utils/ov_hf_downloader.py +++ b/tests/functional/utils/ov_hf_downloader.py @@ -19,6 +19,7 @@ from huggingface_hub import HfApi, snapshot_download from tests.functional.config import huggingface_token +from tests.functional.utils.assertions import OVHfDownloadException from tests.functional.utils.logger import get_logger from tests.functional.utils.test_framework import get_dir_latest_mtime, remove_dir_tree, swap_directory @@ -29,7 +30,7 @@ class OVHfDownloader: def __init__(self, model_type, model_base_path=None): if not huggingface_token: - raise Exception( + raise OVHfDownloadException( "Provide huggingfacace_token with TT_HUGGINGFACE_TOKEN or TT_HUGGINGFACE_TOKEN_FILE_PATH envs" ) self.api = HfApi(token=huggingface_token) diff --git a/tests/functional/utils/test_framework.py b/tests/functional/utils/test_framework.py index 745ee0e250..911c6d342d 100644 --- a/tests/functional/utils/test_framework.py +++ b/tests/functional/utils/test_framework.py @@ -14,6 +14,9 @@ # limitations under the License. # +# pylint: disable=deprecated-argument +# pylint: disable=too-many-positional-arguments + import os import re import shutil diff --git a/tests/requirements.txt b/tests/requirements.txt index 6b4c431f3d..953d1931bc 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,6 +14,7 @@ paramiko==5.0.0 Pillow==12.2.0 protobuf==7.35.1 psutil==7.2.2 +pylint==4.0.6 pytest==9.1.0 pytest-timeout==2.4.0 pytest-xdist==3.8.0 From 2591bb4391c1b196be42d262d939654c365422b8 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Mon, 22 Jun 2026 12:53:18 +0200 Subject: [PATCH 09/13] 20/06/2026 --- tests/functional/conftest.py | 2 +- tests/functional/test_embeddings.py | 115 ----------------------- tests/functional/utils/test_framework.py | 9 +- tests/requirements.txt | 6 +- 4 files changed, 12 insertions(+), 120 deletions(-) delete mode 100644 tests/functional/test_embeddings.py diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 0844f999e0..61cccf30e0 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -54,7 +54,7 @@ if enable_pytest_plugins: - # raise NotImplementedError("OVMS tests not enabled") + raise NotImplementedError("OVMS tests not enabled") pytest_plugins = [ # pylint: disable=unreachable "tests.functional.fixtures.ovms", diff --git a/tests/functional/test_embeddings.py b/tests/functional/test_embeddings.py deleted file mode 100644 index 9c43c4bb53..0000000000 --- a/tests/functional/test_embeddings.py +++ /dev/null @@ -1,115 +0,0 @@ -# -# Copyright (c) 2026 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# pylint: disable=too-many-positional-arguments - -import pytest - -from tests.functional.models.models_library import ModelsLib -from tests.functional.constants.components import OvmsComponents -from tests.functional.constants.ovms_openai import EncodingFormatValues -from tests.functional.constants.ovms_type import OvmsType -from tests.functional.constants.requirements import Requirements -from tests.functional.constants.target_device import TargetDevice -from tests.functional.constants.target_device_configuration import nginx_mtls_not_supported_for_test -from tests.functional.object_model.inference_helpers import run_llm_inference -from tests.functional.utils.context import Context -from tests.functional.utils.generative_ai.utils import calculate_generative_test_timeout, GenerativeAIUtils -from tests.functional.utils.inference.serving.openai import OpenAIWrapper -from tests.functional.utils.logger import get_logger, step -from tests.functional.utils.test_framework import ( - skip_if_language_models_not_enabled, - skip_if_mediapipe_disabled, -) - -logger = get_logger(__name__) - - -@pytest.mark.priority_high -@pytest.mark.components(OvmsComponents.OVMS) -@pytest.mark.reqids(Requirements.embeddings_endpoint, Requirements.openai_api) -@pytest.mark.ovms_types_supported_for_test( - OvmsType.DOCKER, - OvmsType.DOCKER_CMD_LINE, - OvmsType.BINARY, - OvmsType.BINARY_DOCKER, -) -@skip_if_language_models_not_enabled() -@nginx_mtls_not_supported_for_test() -@skip_if_mediapipe_disabled() -class TestEmbeddings: - - @staticmethod - def run_embeddings_endpoints_test( - context: Context, model_type, openai_rest_api_type, endpoint, encoding_format, input_data_type - ): - model, result, port, request_params = GenerativeAIUtils.prepare_resources( - context, - model_type, - openai_rest_api_type, - endpoint, - encoding_format=encoding_format, - ) - - step("Run simple inference") - run_llm_inference( - model, - openai_rest_api_type, - port, - endpoint, - input_data_type=input_data_type, - request_parameters=request_params, - ) - - GenerativeAIUtils.unload_model_and_verify( - model, - result, - port, - openai_rest_api_type, - endpoint, - request_params - ) - - @pytest.mark.api_on_commit - @pytest.mark.devices_supported_for_test(TargetDevice.CPU, TargetDevice.GPU, TargetDevice.NPU) - @pytest.mark.model_type(ModelsLib.various_feature_extraction_models) - @pytest.mark.parametrize("endpoint", [OpenAIWrapper.EMBEDDINGS]) - @pytest.mark.parametrize("encoding_format", EncodingFormatValues.values(), ids=lambda x: f"encoding_format={x}") - @pytest.mark.parametrize("input_data_type", ["list", "string"], ids=lambda x: f"input_data_type={x}") - @pytest.mark.timeout(calculate_generative_test_timeout(480)) - def test_on_commit_embeddings_endpoints( - self, context: Context, model_type, openai_rest_api_type, endpoint, encoding_format, input_data_type - ): - """ - Description: - Execute single inference with LLM type model using embeddings endpoint. - - Input data: - - Language model (feature extraction) type - - Expected results: - OVMS will properly load language model and execute inference. - - Steps: - 1. Prepare language model instance - 2. Start OVMS - 3. Run simple inference - 4. Unload model - 5. Verify model is unreachable - """ - self.run_embeddings_endpoints_test( - context, model_type, openai_rest_api_type, endpoint, encoding_format, input_data_type - ) diff --git a/tests/functional/utils/test_framework.py b/tests/functional/utils/test_framework.py index 911c6d342d..d5a7462726 100644 --- a/tests/functional/utils/test_framework.py +++ b/tests/functional/utils/test_framework.py @@ -21,6 +21,7 @@ import re import shutil import stat +import sys import traceback import pytest @@ -282,8 +283,14 @@ def _make_path_writable_and_retry(func, path, _exc_info): def remove_dir_tree(dir_path, ignore_errors=False): """Remove a directory tree, retrying failed paths after making them writable.""" + # shutil.rmtree accepts the `onexc` callback only on Python 3.12+; older + # interpreters expect `onerror`. Both invoke the same (func, path, *) callback. + if sys.version_info >= (3, 12): + rmtree_kwargs = {"onexc": _make_path_writable_and_retry} + else: + rmtree_kwargs = {"onerror": _make_path_writable_and_retry} try: - shutil.rmtree(dir_path, onerror=_make_path_writable_and_retry) + shutil.rmtree(dir_path, **rmtree_kwargs) except OSError: if not ignore_errors: raise diff --git a/tests/requirements.txt b/tests/requirements.txt index 953d1931bc..84f9cc2e63 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,14 +5,14 @@ distro==1.9.0 docker==7.1.0 filelock==3.29.4 GitPython==3.1.50 -grpcio==1.81.1 +grpcio==1.67.1 Jinja2==3.1.6 jiwer>=4.0.0 openai==2.43.0 opencv-python==4.13.0.92 paramiko==5.0.0 Pillow==12.2.0 -protobuf==7.35.1 +protobuf==6.33.6 psutil==7.2.2 pylint==4.0.6 pytest==9.1.0 @@ -25,7 +25,7 @@ requests-toolbelt==1.0.0 retry==0.9.2 setuptools==81.0.0 soundfile==0.14.0 -tensorboard==2.20.0 # why not in ovms-test +tensorboard==2.20.0 tensorflow==2.21.0 tensorflow-serving-api==2.20.0 tritonclient[all]==2.69.0 From bc6bdfcac2ca3149f32d7bbfcce403b1b7980223 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Mon, 22 Jun 2026 13:08:39 +0200 Subject: [PATCH 10/13] changes to match val repo --- tests/functional/constants/ovms_type.py | 1 - tests/functional/models/models.py | 14 -------------- tests/functional/utils/hooks.py | 6 +----- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/tests/functional/constants/ovms_type.py b/tests/functional/constants/ovms_type.py index 02a1ea9873..458e7c2974 100644 --- a/tests/functional/constants/ovms_type.py +++ b/tests/functional/constants/ovms_type.py @@ -26,7 +26,6 @@ class OvmsType: BINARY_DOCKER = "BINARY_DOCKER" CAPI = "CAPI" CAPI_DOCKER = "CAPI_DOCKER" - KUBERNETES = "KUBERNETES" # legacy # https://github.com/openvinotoolkit/model_server/blob/main/docs/deploying_server.md#deploying-model-server-on-baremetal-without-container diff --git a/tests/functional/models/models.py b/tests/functional/models/models.py index d602a7929c..266e4b154a 100644 --- a/tests/functional/models/models.py +++ b/tests/functional/models/models.py @@ -81,11 +81,7 @@ class ModelInfo: plugin_config: object = None transpose_axes: str = None custom_loader: CustomLoader = None - is_stateful: bool = False expected_batch_size: int = None - max_sequence_number: int = None - idle_sequence_cleanup: bool = None - low_latency_transformation: bool = None base_path: str = None model_path_on_host = None model_type: ModelType = ModelType.IR @@ -158,10 +154,6 @@ def get_list_of_config_fields(): "model_version_policy", "nireq", "plugin_config", - "stateful", - "max_sequence_number", - "idle_sequence_cleanup", - "low_latency_transformation", "allow_cache", ] @@ -217,9 +209,6 @@ def get_config(self): if config_layout: config["layout"] = config_layout - if self.is_stateful: - config["stateful"] = True - return {"config": config} def set_input_shape_for_ovms(self, input_shape: Union[str, List, Dict[str, List]] = None): @@ -379,9 +368,6 @@ def clone(self, clone_model_name=None, model_path_on_host=None): shutil.copytree(self.model_path_on_host, clone.model_path_on_host, dirs_exist_ok=True) return clone - # REMOTE_SERVER_ADDRESS is not pre-defined, but set "on the fly" in our tests (prepare_remote_k8s_cluster_data), - # based on the remote_ip read from the loaded kubeconfig_file (get_ip_from_kubeconfig_file). - def create_new_version(self, container_folder, new_version, copy_from_host_path=False, model_name=None): model_name = model_name if model_name is not None else self.name result = type(self)() diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index 6e8fa6ec93..c91bcacfee 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -131,7 +131,6 @@ TargetDevice.NPU: 1.5, "TRACE_TOOLS": 2, "AUTO_HETERO_MULTI": 3, - OvmsType.KUBERNETES: 1.5, } CURRENT_TARGET_DEVICE_DICT = {} @@ -462,7 +461,7 @@ def get_docker_images(images_to_download): def download_docker_images(): docker_ovms_types = [ - OvmsType.DOCKER, OvmsType.DOCKER_CMD_LINE, OvmsType.KUBERNETES, OvmsType.BINARY_DOCKER, OvmsType.CAPI_DOCKER + OvmsType.DOCKER, OvmsType.DOCKER_CMD_LINE, OvmsType.BINARY_DOCKER, OvmsType.CAPI_DOCKER ] if not any(_ovms_type in docker_ovms_types for _ovms_type in config.ovms_types): return @@ -967,10 +966,7 @@ def apply_conditional_run_type_marks(item): def set_timeout_per_test_type(item, test_type): if item.get_closest_marker("timeout") is None: - ovms_type = item.callspec.params.get(OVMS_TYPE_PARAM_NAME, None) value = timeout_dict[test_type] - if ovms_type == OvmsType.KUBERNETES: - value *= TIMEOUT_MULTIPLIER[OvmsType.KUBERNETES] if any([test_type == MarkRunType.TEST_MARK_REGRESSION, test_type == MarkRunType.TEST_MARK_ON_COMMIT, ]): From 64e7a6c08c3cd444848a1897950861daf2c8513f Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Tue, 23 Jun 2026 13:30:27 +0200 Subject: [PATCH 11/13] [validation_branch=CVS-169692_move_on_commit_tests_ovms_c_repo_part_II] hooks updates --- tests/functional/conftest.py | 179 ++++++++------------- tests/functional/models/models.py | 1 + tests/functional/pylintrc | 21 ++- tests/functional/utils/hooks.py | 38 ++++- tests/functional/utils/ov_hf_downloader.py | 2 +- 5 files changed, 123 insertions(+), 118 deletions(-) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 61cccf30e0..c24176aa63 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -16,37 +16,15 @@ import random import sys -import time -import pytest -from tests.functional.config import enable_pytest_plugins, machine_is_reserved_for_test_session, pytest_keyword_filter -from tests.functional.constants.components import OvmsComponents +from tests.functional.config import enable_pytest_plugins, pytest_keyword_filter, machine_is_reserved_for_test_session from tests.functional.constants.ovms import ( - BASE_OS_PARAM_NAME, CURRENT_TARGET_DEVICE_DICT_ARGUMENT, - OVMS_TYPE_PARAM_NAME, - TARGET_DEVICE_PARAM_NAME, TMP_REPOS_DIR_ARGUMENT, - USES_MAPPING_PARAM_NAME, ) from tests.functional.utils import hooks -from tests.functional.utils.hooks import ( - log_configuration_variables, - parametrize_all_models, - parametrize_base_os, - parametrize_input_shape, - parametrize_iteration_info, - parametrize_many_models, - parametrize_model_aux_type, - parametrize_model_type, - parametrize_ovms_type, - parametrize_plugin_config, - parametrize_target_device, - parametrize_uses_mapping, - validate_port_pool, -) from tests.functional.utils.logger import OvmsFileHandler, get_logger -from tests.functional.utils.marks import MarksRegistry, MarkTestParameters +from tests.functional.utils.marks import MarksRegistry from tests.functional.utils.test_framework import is_xdist_master logger = get_logger(__name__) @@ -86,7 +64,7 @@ def pytest_configure(config): if is_xdist_master(): hooks.setup_tmp_repos_dir(config) - validate_port_pool(config) + hooks.validate_port_pool(config) # master thread pytest_configure call. No xdist worker process spawned yet. hooks.init_environment(config) hooks.clear_ovms_capi_artifacts() @@ -133,99 +111,70 @@ def pytest_configure_node(node): MarksRegistry.MARK_ENUMS.extend([OvmsComponents]) -def pytest_sessionstart(session): - logger.info(f"Starting test session in the following folder: {session.startdir}") - log_configuration_variables() - session.start_time = time.time() + def pytest_sessionstart(session): + hooks.get_session_start_info(session) + + + @pytest.hookimpl(hookwrapper=True) + def pytest_collection_modifyitems(session, config, items): + """ + Support for running tests with component tags. + Report all test component markers to mongo_reporter. + """ + logger.info(f"Preparing tests for test session in the following folder: {session.startdir}") + + if pytest_keyword_filter: + # Filter case insensitive + deselected = [_item for _item in items if pytest_keyword_filter.lower() not in _item.name.lower()] + if deselected: + hooks.deselect_items(items, config, deselected) + yield # deselect items in default hook way via keyword ('-k') -# https://docs.pytest.org/en/6.2.x/reference.html#id57 -@pytest.hookimpl(hookwrapper=True) -def pytest_collection_modifyitems(session, config, items): - """ - Support for running tests with component tags. - Report all test component markers to mongo_reporter. - """ - logger.info(f"Preparing tests for test session in the following folder: {session.startdir}") + if config.option.collectonly: + hooks.log_skip_statistic(items) - if pytest_keyword_filter: - # Filter case insensitive - deselected = [_item for _item in items if pytest_keyword_filter.lower() not in _item.name.lower()] + deselected = hooks.preprocess_collected_items(items) if deselected: hooks.deselect_items(items, config, deselected) - yield # deselect items in default hook way via keyword ('-k') - - if config.option.collectonly: - hooks.log_skip_statistic(items) - - deselected = hooks.preprocess_collected_items(items) - if deselected: - hooks.deselect_items(items, config, deselected) - - hooks.set_divide_target_device_per_worker(items) - - random.Random(7).shuffle(items) - - -# https://docs.pytest.org/en/6.2.x/reference.html#id58 -@pytest.hookimpl(hookwrapper=True, tryfirst=True) -def pytest_runtest_protocol(item: "Item"): - """ - Perform the runtest protocol for a single test item. - The default runtest protocol is this (see individual hooks for full details): - pytest_runtest_logstart(nodeid, location) - Setup phase: - call = pytest_runtest_setup(item) (wrapped in CallInfo(when="setup")) - report = pytest_runtest_makereport(item, call) - pytest_runtest_logreport(report) - pytest_exception_interact(call, report) if an interactive exception occurred - Call phase, if the setup passed and the setuponly pytest option is not set: - call = pytest_runtest_call(item) (wrapped in CallInfo(when="call")) - report = pytest_runtest_makereport(item, call) - pytest_runtest_logreport(report) - pytest_exception_interact(call, report) if an interactive exception occurred - Teardown phase: - call = pytest_runtest_teardown(item, nextitem) (wrapped in CallInfo(when="teardown")) - report = pytest_runtest_makereport(item, call) - pytest_runtest_logreport(report) - pytest_exception_interact(call, report) if an interactive exception occurred - pytest_runtest_logfinish(nodeid, location) - """ - __root_logger = get_logger(None) - if not item.keywords.get("skip"): - fh = OvmsFileHandler(item) - __root_logger.addHandler(fh) - yield - if not item.keywords.get("skip"): - fh.close() - __root_logger.removeHandler(fh) - - -def pytest_generate_tests(metafunc): - if OVMS_TYPE_PARAM_NAME in metafunc.fixturenames: - parametrize_ovms_type(metafunc) - - if USES_MAPPING_PARAM_NAME in metafunc.fixturenames: - parametrize_uses_mapping(metafunc) - - if BASE_OS_PARAM_NAME in metafunc.fixturenames: - parametrize_base_os(metafunc) - - if MarkTestParameters.MODEL_TYPE in metafunc.fixturenames: - parametrize_model_type(metafunc) - elif MarkTestParameters.ALL_MODELS in metafunc.fixturenames: - parametrize_all_models(metafunc) - elif MarkTestParameters.MANY_MODELS in metafunc.fixturenames: - parametrize_many_models(metafunc) - elif MarkTestParameters.ITERATION_INFO in metafunc.fixturenames: - parametrize_iteration_info(metafunc) - elif MarkTestParameters.INPUT_SHAPE in metafunc.fixturenames: - parametrize_input_shape(metafunc) - elif MarkTestParameters.PLUGIN_CONFIG in metafunc.fixturenames: - parametrize_plugin_config(metafunc) - elif TARGET_DEVICE_PARAM_NAME in metafunc.fixturenames: - parametrize_target_device(metafunc) - - if MarkTestParameters.MODEL_AUX_TYPE in metafunc.fixturenames: - parametrize_model_aux_type(metafunc) + hooks.set_divide_target_device_per_worker(items) + + random.Random(7).shuffle(items) + + + @pytest.hookimpl(hookwrapper=True, tryfirst=True) + def pytest_runtest_protocol(item: "Item"): + """ + Perform the runtest protocol for a single test item. + The default runtest protocol is this (see individual hooks for full details): + pytest_runtest_logstart(nodeid, location) + Setup phase: + call = pytest_runtest_setup(item) (wrapped in CallInfo(when="setup")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + Call phase, if the setup passed and the setuponly pytest option is not set: + call = pytest_runtest_call(item) (wrapped in CallInfo(when="call")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + Teardown phase: + call = pytest_runtest_teardown(item, nextitem) (wrapped in CallInfo(when="teardown")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + pytest_runtest_logfinish(nodeid, location) + """ + __root_logger = get_logger(None) + if not item.keywords.get("skip"): + fh = OvmsFileHandler(item) + __root_logger.addHandler(fh) + yield + if not item.keywords.get("skip"): + fh.close() + __root_logger.removeHandler(fh) + + + def pytest_generate_tests(metafunc): + hooks.parametrize_tests(metafunc) diff --git a/tests/functional/models/models.py b/tests/functional/models/models.py index 266e4b154a..d5a569c0da 100644 --- a/tests/functional/models/models.py +++ b/tests/functional/models/models.py @@ -98,6 +98,7 @@ class ModelInfo: use_subconfig: bool = False xml_name: str = None onnx_name: str = None + is_generative: bool = False is_llm: bool = False is_vision_language: bool = False is_hf_direct_load: bool = False # model can be loaded directly from HuggingFace without conversion with optimum-cli diff --git a/tests/functional/pylintrc b/tests/functional/pylintrc index 346b484f5c..90c83a19e1 100644 --- a/tests/functional/pylintrc +++ b/tests/functional/pylintrc @@ -1,3 +1,19 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + [MAIN] # Analyse import fallback blocks. This can be used to support both Python 2 and @@ -170,7 +186,7 @@ persistent=yes # Minimum Python version to use for version dependent checks. Will default to # the version used to run pylint. -py-version=3.10 +py-version=3.12 # Discover python modules and packages in the file system subtree. recursive=yes @@ -364,6 +380,9 @@ ignored-parents= # Maximum number of arguments for function / method. max-args=8 +# Maximum number of positional arguments for function / method. +max-positional-arguments=20 + # Maximum number of attributes for a class (see R0902). max-attributes=7 diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index c91bcacfee..879bd2e560 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -19,6 +19,7 @@ import re import shutil import sys +import time import warnings import pytest @@ -772,7 +773,7 @@ def mute_warnings(): def setup_nginx(): if not is_nginx_mtls: return - print("Setup ngnix certificates") + print("Setup nginx certificates") if is_xdist_master(): OvsaCerts.generate_ovsa_certs(skip_if_valid=not force_generate_new_ssl_certs) OvsaCerts.init_ovsa_certs() @@ -1112,3 +1113,38 @@ def log_skip_statistic(items): issue_stats, other_tests = calc_statistics(items) log_labeled_stats(issue_stats) log_others(other_tests) + + +def get_session_start_info(session): + logger.info(f"Starting test session in the following folder: {session.startdir}") + log_configuration_variables() + session.start_time = time.time() + + +def parametrize_tests(metafunc): + if OVMS_TYPE_PARAM_NAME in metafunc.fixturenames: + parametrize_ovms_type(metafunc) + + if USES_MAPPING_PARAM_NAME in metafunc.fixturenames: + parametrize_uses_mapping(metafunc) + + if BASE_OS_PARAM_NAME in metafunc.fixturenames: + parametrize_base_os(metafunc) + + if MarkTestParameters.MODEL_TYPE in metafunc.fixturenames: + parametrize_model_type(metafunc) + elif MarkTestParameters.ALL_MODELS in metafunc.fixturenames: + parametrize_all_models(metafunc) + elif MarkTestParameters.MANY_MODELS in metafunc.fixturenames: + parametrize_many_models(metafunc) + elif MarkTestParameters.ITERATION_INFO in metafunc.fixturenames: + parametrize_iteration_info(metafunc) + elif MarkTestParameters.INPUT_SHAPE in metafunc.fixturenames: + parametrize_input_shape(metafunc) + elif MarkTestParameters.PLUGIN_CONFIG in metafunc.fixturenames: + parametrize_plugin_config(metafunc) + elif TARGET_DEVICE_PARAM_NAME in metafunc.fixturenames: + parametrize_target_device(metafunc) + + if MarkTestParameters.MODEL_AUX_TYPE in metafunc.fixturenames: + parametrize_model_aux_type(metafunc) diff --git a/tests/functional/utils/ov_hf_downloader.py b/tests/functional/utils/ov_hf_downloader.py index cc2616e21d..f020c16f59 100644 --- a/tests/functional/utils/ov_hf_downloader.py +++ b/tests/functional/utils/ov_hf_downloader.py @@ -31,7 +31,7 @@ class OVHfDownloader: def __init__(self, model_type, model_base_path=None): if not huggingface_token: raise OVHfDownloadException( - "Provide huggingfacace_token with TT_HUGGINGFACE_TOKEN or TT_HUGGINGFACE_TOKEN_FILE_PATH envs" + "Provide huggingface_token with TT_HUGGINGFACE_TOKEN or TT_HUGGINGFACE_TOKEN_FILE_PATH envs" ) self.api = HfApi(token=huggingface_token) self.model = model_type() From 65655b494e2aa7a7cabfe92827b238916bceeb7b Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Tue, 23 Jun 2026 16:03:52 +0200 Subject: [PATCH 12/13] [validation_branch=CVS-169692_move_on_commit_tests_ovms_c_repo_part_II] updates --- ci/build_test_OnCommit.groovy | 2 +- tests/functional/config.py | 2 +- tests/functional/conftest.py | 2 +- tests/functional/models/models.py | 2 +- tests/functional/object_model/ovms_binary.py | 2 +- tests/functional/object_model/ovms_params.py | 2 +- tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu | 2 +- tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/build_test_OnCommit.groovy b/ci/build_test_OnCommit.groovy index fc9f4b8ead..863f074895 100644 --- a/ci/build_test_OnCommit.groovy +++ b/ci/build_test_OnCommit.groovy @@ -408,7 +408,7 @@ pipeline { def ovms_c_repo_path = bat(returnStdout: true, script: 'cd .. && cd').trim().split('\n').last().trim() def cmd_link_ovms = "(if exist ${current_path}\\tests\\functional rmdir ${current_path}\\tests\\functional) && mklink /D ${current_path}\\tests\\functional ${ovms_c_repo_path}\\tests\\functional" def cmd_requirements = "(if not exist .venv virtualenv .venv --python=python3.12) && call .venv\\Scripts\\activate.bat && pip install -r requirements.txt" - def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && \"TT_LOGGING_LEVEL_OVMS=DEBUG\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\"" + def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_LOGGING_LEVEL_OVMS=DEBUG\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\"" def cmd_pytest = "pytest tests/non_functional/documentation -k \"${test_doc_files_str}\" -n 0 --dist loadgroup --basetemp=\"C:\\tmp\\pytest-${BRANCH_NAME}-${BUILD_NUMBER}\"" def cmd = "" if ( win_image_build_needed == "true" ) { diff --git a/tests/functional/config.py b/tests/functional/config.py index c21f3ddf67..df8d71c843 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -323,7 +323,7 @@ def get_uses_mapping(): __base_os = os.environ.get("BASE_OS", OsType.Ubuntu24) base_os = get_list("TT_BASE_OS", fallback=[__base_os]) -"""" BASE_IMAGE - Docker image used during OVMS image creation""" +""" BASE_IMAGE - Docker image used during OVMS image creation """ base_image = os.environ.get("BASE_IMAGE", None) if base_image is not None: assert len(base_os) == 1, "If you wish to iterate by TT_BASE_OS: do not set BASE_IMAGE explicitly." diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index c24176aa63..1bb8d68b10 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -86,7 +86,7 @@ def pytest_configure(config): def pytest_unconfigure(config): if getattr(config, "configured", None) is not True: - # Check if pytest_configure() was done successfuly, if not: logger would be in invalid state so disable. + # Check if pytest_configure() was done successfully, if not: logger would be in invalid state so disable. for _logger in logger.manager.loggerDict.values(): _logger.disabled = True diff --git a/tests/functional/models/models.py b/tests/functional/models/models.py index d5a569c0da..cad6f47ae5 100644 --- a/tests/functional/models/models.py +++ b/tests/functional/models/models.py @@ -593,7 +593,7 @@ def is_dynamic(self): return False @staticmethod - def rename_input_ouput_data(data, src_name, dst_name): + def rename_input_output_data(data, src_name, dst_name): data[dst_name] = data[src_name] del data[src_name] return data diff --git a/tests/functional/object_model/ovms_binary.py b/tests/functional/object_model/ovms_binary.py index adec1fdf5f..1d4ef610ac 100644 --- a/tests/functional/object_model/ovms_binary.py +++ b/tests/functional/object_model/ovms_binary.py @@ -100,7 +100,7 @@ def start_binary_ovms( cpu_extension_path = None if parameters.models is not None: for model in parameters.models: - if model.cpu_extension is not None: + if getattr(model, "cpu_extension", None) is not None: cpu_extension = model.cpu_extension() cpu_extension_path = cpu_extension.lib_path[1:] if parameters.cpu_extension: diff --git a/tests/functional/object_model/ovms_params.py b/tests/functional/object_model/ovms_params.py index f1b1c7368e..e732516f4b 100644 --- a/tests/functional/object_model/ovms_params.py +++ b/tests/functional/object_model/ovms_params.py @@ -83,7 +83,7 @@ class OvmsParams(object): def __post_init__(self): if self.models is not None: for model in self.models: - if model.cpu_extension is not None: + if getattr(model, "cpu_extension", None) is not None: self.cpu_extension = model.cpu_extension() break diff --git a/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu b/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu index 2aba99d708..bdc6bab436 100644 --- a/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu +++ b/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu @@ -22,7 +22,7 @@ FROM $OVMS_IMAGE as ovms_image FROM $OVMS_TEST_IMAGE as ovms_test_image FROM $BASE_IMAGE as base_image -ARG OVMS_DEPENDENCIES="libcurl4-openssl-dev libpugixml1v5 libtbb2 libxml12" +ARG OVMS_DEPENDENCIES="libcurl4-openssl-dev libpugixml1v5 libtbb12 libxml2" RUN apt-get update && apt-get install -y --no-install-recommends ${OVMS_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* WORKDIR / diff --git a/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu b/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu index 1fb7d4b5c7..35a2007078 100644 --- a/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu +++ b/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu @@ -22,7 +22,7 @@ FROM $OVMS_IMAGE as ovms_image FROM $OVMS_TEST_IMAGE as ovms_test_image FROM $BASE_IMAGE as base_image -ARG OVMS_DEPENDENCIES="libcurl4-openssl-dev libpugixml1v5 libtbb2 libxml12" +ARG OVMS_DEPENDENCIES="libcurl4-openssl-dev libpugixml1v5 libtbb12 libxml2" RUN apt-get update && apt-get install -y --no-install-recommends ${OVMS_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* WORKDIR / From 4f28c0d87a16a4effa7485c880751640ff8c9ad6 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Wed, 24 Jun 2026 13:57:16 +0200 Subject: [PATCH 13/13] [validation_branch=CVS-169692_move_on_commit_tests_ovms_c_repo_part_II] updates to match validation repo --- tests/functional/utils/ov_hf_downloader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/utils/ov_hf_downloader.py b/tests/functional/utils/ov_hf_downloader.py index f020c16f59..ab61475b38 100644 --- a/tests/functional/utils/ov_hf_downloader.py +++ b/tests/functional/utils/ov_hf_downloader.py @@ -48,7 +48,7 @@ def check_and_update_hf_model(self): if local_latest_dt and repo_info.last_modified <= local_latest_dt: print(f"No files to update for model: {self.model_name}") - return + return False print(f"Download OVHf model: {self.model_name}") staging_path = self.model_local_path + "_staging" @@ -56,6 +56,7 @@ def check_and_update_hf_model(self): remove_dir_tree(staging_path) self.download_model(model_dir=staging_path) swap_directory(self.model_local_path, staging_path) + return True def download_model(self, model_name=None, model_dir=None, force_download=False): snapshot_download(