diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1bd89a2d1..610e33c7d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -53,6 +53,7 @@ jobs: -DWITH_LUA=OFF -DWITH_OPENMP=OFF -DWITH_PLUGINS=OFF + -DWITH_PYBIND=OFF -DWITH_SRC=OFF -DWITH_TESTS=OFF -DWITH_TOOLS=OFF @@ -90,6 +91,11 @@ jobs: - name: Build run: cmake --build build ${{ env.CMAKE_BUILD_OPTS }} + # WITH_PYBIND is silently disabled when pybind11 is missing + - name: Verify the Python bindings were built + if: matrix.distro == 'fedora' + run: test -f build/python/binding/_core*.so + - name: Upload Build Artifacts uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2d9b52b3d..f94ad23b0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -110,6 +110,15 @@ jobs: ref: ${{ needs.setup.outputs.ref }} ref_name: ${{ needs.setup.outputs.ref_name }} + packaging-python: + name: Build Python Wheels + needs: [setup] + secrets: inherit + uses: ./.github/workflows/packaging-python.yaml + with: + container_image: ${{ needs.setup.outputs.container_image }} + ref: ${{ needs.setup.outputs.ref }} + packaging-nix: name: Bundle Artifacts with Nix needs: [setup, nix] @@ -162,6 +171,7 @@ jobs: - test - packaging-container - packaging-nix + - packaging-python - deploy - deploy-nix runs-on: ubuntu-latest diff --git a/.github/workflows/packaging-python.yaml b/.github/workflows/packaging-python.yaml new file mode 100644 index 000000000..33202e219 --- /dev/null +++ b/.github/workflows/packaging-python.yaml @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-License-Identifier: Apache-2.0 + +name: Packaging (Python) + +on: + workflow_call: + inputs: + container_image: + required: true + type: string + ref: + description: 'Revision under test' + required: true + type: string + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + sdist: + name: Source distribution + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.sha }} + allow-unsafe-pr-checkout: true + submodules: recursive + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Build source distribution + run: pipx run build --sdist --outdir dist/ + + - name: Archive source distribution + uses: actions/upload-artifact@v7 + with: + name: python-sdist + path: dist/ + + image: + name: Prepare Wheel Build Image + runs-on: ubuntu-latest + outputs: + image: ${{ steps.image.outputs.image }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.sha }} + allow-unsafe-pr-checkout: true + submodules: recursive + + - name: Login to Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Set up Buildx + uses: docker/setup-buildx-action@v4 + + # Tagged by recipe, so the image is built once and pulled afterwards + - name: Compute image name + id: image + env: + CONTAINER_IMAGE: ${{ inputs.container_image }} + REPOSITORY: ${{ github.repository }} + run: | + HASH=$(cat packaging/docker/Dockerfile.manylinux packaging/deps.sh \ + $(find packaging/patches -type f | sort) | sha256sum | cut -c1-16) + echo "image=${CONTAINER_IMAGE:-ghcr.io/${REPOSITORY,,}}/manylinux:${HASH}" >> "$GITHUB_OUTPUT" + echo "latest=${CONTAINER_IMAGE:-ghcr.io/${REPOSITORY,,}}/manylinux:latest" >> "$GITHUB_OUTPUT" + + - name: Check for an existing image + id: existing + env: + IMAGE: ${{ steps.image.outputs.image }} + run: | + if docker buildx imagetools inspect "${IMAGE}" >/dev/null 2>&1; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build and Push Image + if: steps.existing.outputs.found == 'false' + uses: docker/build-push-action@v7 + with: + context: . + file: packaging/docker/Dockerfile.manylinux + push: true + cache-from: type=gha,scope=manylinux + cache-to: ${{ github.event_name != 'pull_request_target' && 'type=gha,mode=max,scope=manylinux' || '' }} + tags: | + ${{ steps.image.outputs.image }} + ${{ github.event_name == 'push' && steps.image.outputs.latest || '' }} + + wheels: + name: Wheel (${{ matrix.python }}, ${{ matrix.variant }}) + needs: [image] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: [cp310, cp311, cp312, cp313, cp314] + variant: [apache, gpl] + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.sha }} + allow-unsafe-pr-checkout: true + submodules: recursive + + - name: Login to Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Build wheel + uses: pypa/cibuildwheel@v4.2.0 + env: + CIBW_BUILD: ${{ matrix.python }}-manylinux_x86_64 + CIBW_MANYLINUX_X86_64_IMAGE: ${{ needs.image.outputs.image }} + CIBW_ENVIRONMENT_PASS_LINUX: SKBUILD_CMAKE_DEFINE VILLAS_WHEEL_VARIANT + SKBUILD_CMAKE_DEFINE: ${{ matrix.variant == 'gpl' && 'WITHOUT_GPL=OFF' || '' }} + VILLAS_WHEEL_VARIANT: ${{ matrix.variant }} + + - name: Archive wheel + uses: actions/upload-artifact@v7 + with: + name: python-wheel-${{ matrix.variant }}-${{ matrix.python }} + path: wheelhouse/ diff --git a/CMakeLists.txt b/CMakeLists.txt index e0bc467b8..b78b99a0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,6 +94,8 @@ find_package(Criterion) find_package(OpalOrchestra) find_package(LibXml2) find_package(OpalAsyncApi) +find_package(Python3 COMPONENTS Interpreter Development.Module) +find_package(pybind11 CONFIG QUIET) # Check for tools find_program(PROTOBUFC_COMPILER NAMES protoc-c) @@ -190,6 +192,7 @@ cmake_dependent_option(WITH_HOOKS "Build with support for processi cmake_dependent_option(WITH_LUA "Build with Lua" "${WITH_DEFAULTS}" "LUA_FOUND" OFF) cmake_dependent_option(WITH_OPENMP "Build with support for OpenMP for parallel hooks" "${WITH_DEFAULTS}" "OPENMP_FOUND" OFF) cmake_dependent_option(WITH_PLUGINS "Build plugins" "${WITH_DEFAULTS}" "TOPLEVEL_PROJECT" OFF) +cmake_dependent_option(WITH_PYBIND "Build Python bindings" "${WITH_DEFAULTS}" "pybind11_FOUND; Python3_FOUND" OFF) cmake_dependent_option(WITH_SRC "Build executables" "${WITH_DEFAULTS}" "TOPLEVEL_PROJECT" OFF) cmake_dependent_option(WITH_TESTS "Run tests" "${WITH_DEFAULTS}" "TOPLEVEL_PROJECT" OFF) cmake_dependent_option(WITH_TOOLS "Build auxilary tools" "${WITH_DEFAULTS}" "TOPLEVEL_PROJECT" OFF) @@ -301,6 +304,7 @@ add_feature_info(HOOKS WITH_HOOKS "Build with add_feature_info(LUA WITH_LUA "Build with Lua support") add_feature_info(OPENMP WITH_OPENMP "Build with OpenMP support") add_feature_info(PLUGINS WITH_PLUGINS "Build plugins") +add_feature_info(PYBIND WITH_PYBIND "Build Python bindings") add_feature_info(SRC WITH_SRC "Build executables") add_feature_info(TESTS WITH_TESTS "Run tests") add_feature_info(TOOLS WITH_TOOLS "Build auxilary tools") diff --git a/flake.nix b/flake.nix index 84abd169a..daaaf986b 100644 --- a/flake.nix +++ b/flake.nix @@ -66,7 +66,10 @@ packagesWith = pkgs: rec { default = villas-node; - villas-node-python = pkgs.callPackage (nixDir + "/python.nix") { src = ./.; }; + villas-node-python = pkgs.callPackage (nixDir + "/python.nix") { + src = ./.; + villas = villas-node-minimal; + }; villas-node-minimal = pkgs.callPackage (nixDir + "/villas.nix") { src = ./.; diff --git a/lib/hook_list.cpp b/lib/hook_list.cpp index ed5db006d..ffb24862b 100644 --- a/lib/hook_list.cpp +++ b/lib/hook_list.cpp @@ -189,6 +189,9 @@ void HookList::dump(Logger logger, std::string subject) const { logger->debug("Hooks of {}:", subject); unsigned i = 0; - for (auto h : *this) - logger->debug(" {}: {}", i++, h->getFactory()->getName()); + for (auto h : *this) { + auto *f = h->getFactory(); + + logger->debug(" {}: {}", i++, f ? f->getName() : "unnamed"); + } } diff --git a/lib/node.cpp b/lib/node.cpp index 59718db6b..8a6cdf381 100644 --- a/lib/node.cpp +++ b/lib/node.cpp @@ -440,7 +440,7 @@ Node *NodeFactory::make(json_t *json, const uuid_t &id, std::string type; Node *n; - if (json_is_object(json)) + if (!json_is_object(json)) throw ConfigError(json, "node-config-node", "Node configuration must be an object"); diff --git a/lib/node_capi.cpp b/lib/node_capi.cpp index ab96b1bdc..446aad70a 100644 --- a/lib/node_capi.cpp +++ b/lib/node_capi.cpp @@ -59,7 +59,7 @@ int node_restart(vnode *n) { int node_destroy(vnode *n) { auto *nc = (Node *)n; - nc->~Node(); + delete nc; return 0; } diff --git a/packaging/check-wheel.py b/packaging/check-wheel.py new file mode 100644 index 000000000..c0b00f8b3 --- /dev/null +++ b/packaging/check-wheel.py @@ -0,0 +1,75 @@ +""" +SPDX-FileCopyrightText: 2026 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import os + +import villas.node as vn + +# Node-types disappear silently when a dependency is missing at configure time +REQUIRED = { + "amqp", + "c37.118", + "can", + "comedi", + "exec", + "file", + "infiniband", + "influxdb", + "kafka", + "loopback", + "modbus", + "mqtt", + "nanomsg", + "ngsi", + "redis", + "rtp", + "shmem", + "signal.v2", + "socket", + "stats", + "temper", + "test_rtt", + "uldaq", + "webrtc", + "websocket", +} + +GPL = { + "ethercat", + "iec60870-5-104", + "iec61850-8-1", + "iec61850-9-2", + "zeromq", +} + +gpl = os.environ.get("VILLAS_WHEEL_VARIANT") == "gpl" + +types = set(vn.node_types()) + +missing = sorted((REQUIRED | GPL if gpl else REQUIRED) - types) +if missing: + raise SystemExit(f"node-types missing from the wheel: {missing}") + +if not gpl and GPL & types: + raise SystemExit(f"GPL node-types in an Apache-2.0 wheel: {sorted(GPL & types)}") + +config = { + "type": "signal.v2", + "limit": 1, + "rate": 100.0, + "in": {"signals": [{"name": "sine", "signal": "sine"}]}, +} + +node = vn.Node(json.dumps(config), "check") +node.check() +node.prepare() +node.start() +sample = node.read()[0] +node.stop() + +print( + f"{len(types)} node-types, {len(vn.hook_types())} hooks, read {sample.length} value" +) diff --git a/packaging/docker/Dockerfile.fedora b/packaging/docker/Dockerfile.fedora index df6e516f4..5418a02a9 100644 --- a/packaging/docker/Dockerfile.fedora +++ b/packaging/docker/Dockerfile.fedora @@ -28,7 +28,7 @@ RUN dnf -y install \ openssh-clients \ jq nmap-ncat \ iproute iproute-tc \ - python python-devel python-pip \ + python python-devel python-pip pybind11-devel \ gdb gdb-gdbserver \ cppcheck \ xmlto dblatex rubygem-asciidoctor \ @@ -85,8 +85,9 @@ RUN bash /deps/deps.sh RUN ln -s /usr/lib64/tc /usr/lib/tc # Install Python dev tools for CI -COPY ./python /python -RUN pip install /python[dev] && \ +COPY ./pyproject.toml ./LICENSE /python/ +COPY ./python /python/python +RUN pip install --config-settings=wheel.cmake=false "/python[dev]" && \ rm -r /python # Expose ports for HTTP and WebSocket frontend diff --git a/packaging/docker/Dockerfile.manylinux b/packaging/docker/Dockerfile.manylinux new file mode 100644 index 000000000..6b76df7c0 --- /dev/null +++ b/packaging/docker/Dockerfile.manylinux @@ -0,0 +1,51 @@ +# Base image for building VILLASnode Python wheels with cibuildwheel +# +# SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-License-Identifier: Apache-2.0 + +FROM quay.io/pypa/manylinux_2_28_x86_64 + +LABEL \ + org.opencontainers.image.title="VILLASnode manylinux" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.url="http://fein-aachen.org/projects/villas-framework/" \ + org.opencontainers.image.source="https://github.com/VILLASframework/node" + +RUN dnf -y install epel-release && \ + dnf -y install \ + autoconf \ + automake \ + bison \ + flex \ + git \ + graphviz-devel \ + jansson-devel \ + libconfig-devel \ + libcurl-devel \ + libibverbs-devel \ + libmodbus-devel \ + libnice-devel \ + libnl3-devel \ + librdkafka-devel \ + librdmacm-devel \ + libtool \ + libusb-devel \ + libuuid-devel \ + lua-devel \ + make \ + mosquitto-devel \ + nanomsg-devel \ + openssl-devel \ + patch \ + pkgconfig \ + protobuf-c-devel \ + protobuf-devel \ + subversion \ + wget \ + zeromq-devel + +ADD packaging/patches /deps/patches +ADD packaging/deps.sh /deps +RUN DEPS_NONINTERACTIVE=1 \ + DEPS_SKIP=criterion \ + bash /deps/deps.sh diff --git a/packaging/nix/python.nix b/packaging/nix/python.nix index d91af6ef0..67719babd 100644 --- a/packaging/nix/python.nix +++ b/packaging/nix/python.nix @@ -4,17 +4,24 @@ src, pkgs, python3Packages, + villas, }: python3Packages.buildPythonPackage { name = "villas-node"; - src = "${src}/python"; + inherit src; format = "pyproject"; + dontUseCmakeConfigure = true; + nativeBuildInputs = villas.nativeBuildInputs ++ [ python3Packages.pybind11 ]; + buildInputs = villas.buildInputs; propagatedBuildInputs = with python3Packages; [ linuxfd requests protobuf ]; - build-system = with python3Packages; [ setuptools ]; + build-system = with python3Packages; [ + scikit-build-core + pybind11 + ]; nativeCheckInputs = with python3Packages; [ black flake8 @@ -28,6 +35,6 @@ python3Packages.buildPythonPackage { ]; postPatch = '' - ${pkgs.protobuf}/bin/protoc --proto_path ${src}/lib/formats --mypy_out=villas/node --python_out=villas/node/ ${src}/lib/formats/villas.proto + ${pkgs.protobuf}/bin/protoc --proto_path ${src}/lib/formats --mypy_out=python/villas/node --python_out=python/villas/node/ ${src}/lib/formats/villas.proto ''; } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..6580fa3fb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: 2023 OPAL-RT Germany GmbH +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ['scikit-build-core>=0.10', 'pybind11>=2.12', 'pybind11-stubgen>=2.5'] +build-backend = 'scikit_build_core.build' + +[project] +name = 'villas-node' +version = "1.2.0" +description = 'Python support for the VILLASnode simulation-data gateway' +readme = 'python/README.md' +requires-python = '>=3.10' +keywords = ['simulation', 'power', 'system', 'real-time', 'villas'] +classifiers = [ + 'Development Status :: 4 - Beta', + 'Topic :: Scientific/Engineering', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3', +] +dependencies = [ + 'linuxfd==1.5; platform_system=="Linux"', + 'protobuf==6.32.1', + 'libconf==2.0.1', +] +optional-dependencies.dev = [ + 'black==25.11.0', + 'flake8==7.3.0', + 'mypy==1.19.0', + 'pytest==9.0.1', + 'types-protobuf==6.32.1.20251105', +] + +[project.scripts] +villas-conf2orchestra-ddf = "villas.node.opal_orchestra_ddf:main" + +[project.urls] +GitHub = 'https://github.com/VILLASframework/node' +Project = 'https://www.fein-aachen.org/en/projects/villas-node' +Homepage = 'https://villas.fein-aachen.org' + +[[project.authors]] +name = 'Steffen Vogel' +email = 'post@steffenvogel.de' + +[[project.authors]] +name = 'Philipp Jungkamp' +email = 'Philipp.Jungkamp@opal-rt.com' + +[tool.scikit-build] +cmake.build-type = 'Release' +build.targets = ['python-bindings'] +wheel.packages = ['python/villas'] +wheel.exclude = ['**/test_*.py'] +install.components = ['lib', 'python'] +wheel.license-files = ['LICENSE'] +sdist.exclude = ['.github', 'doc', 'paper', 'web'] + +[tool.scikit-build.cmake.define] +WITH_PYBIND = 'ON' +WITH_CLIENTS = 'OFF' +WITH_FPGA = 'OFF' +WITH_PLUGINS = 'OFF' +WITH_SRC = 'OFF' +WITH_TESTS = 'OFF' +WITH_TOOLS = 'OFF' +WITHOUT_GPL = 'ON' +VILLAS_PYTHON_INSTALL_DIR = '.' +CMAKE_INSTALL_LIBDIR = 'villas/node' +CMAKE_INSTALL_RPATH = '$ORIGIN' + +[tool.cibuildwheel] +build = 'cp3{10,11,12,13,14}-manylinux_x86_64' +build-verbosity = 1 +test-command = 'python {project}/packaging/check-wheel.py' + +[tool.cibuildwheel.linux] +manylinux-x86_64-image = 'ghcr.io/villasframework/node/manylinux:latest' +environment = { LD_LIBRARY_PATH = '/usr/local/lib:/usr/local/lib64', PKG_CONFIG_PATH = '/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig:/usr/local/share/pkgconfig' } diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f9aa07f2f..7b1792645 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -20,3 +20,7 @@ if(DEFINED PROTOBUF_COMPILER AND PROTOBUF_FOUND) villas_pb2.py ) endif() + +if(WITH_PYBIND) + add_subdirectory(binding) +endif() diff --git a/python/binding/CMakeLists.txt b/python/binding/CMakeLists.txt new file mode 100644 index 000000000..29bcc0529 --- /dev/null +++ b/python/binding/CMakeLists.txt @@ -0,0 +1,50 @@ +# CMakeLists. +# +# SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-License-Identifier: Apache-2.0 + +set(VILLAS_PYTHON_INSTALL_DIR + "${CMAKE_INSTALL_LIBDIR}/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/site-packages" + CACHE STRING "Installation directory for the Python package" +) + +pybind11_add_module(python-bindings module.cpp) + +target_link_libraries(python-bindings PRIVATE villas) + +set_target_properties(python-bindings PROPERTIES + OUTPUT_NAME _core + INSTALL_RPATH "$ORIGIN" +) + +execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import pybind11_stubgen" + RESULT_VARIABLE PYBIND11_STUBGEN_MISSING + OUTPUT_QUIET + ERROR_QUIET +) + +if(NOT PYBIND11_STUBGEN_MISSING) + add_custom_command(TARGET python-bindings POST_BUILD + COMMAND ${CMAKE_COMMAND} -E env + PYTHONPATH=$ + ${Python3_EXECUTABLE} -m pybind11_stubgen _core + --output-dir ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/py.typed + VERBATIM + ) + + install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/_core.pyi + ${CMAKE_CURRENT_BINARY_DIR}/py.typed + COMPONENT python + DESTINATION ${VILLAS_PYTHON_INSTALL_DIR}/villas/node + ) +endif() + +install( + TARGETS python-bindings + COMPONENT python + LIBRARY DESTINATION ${VILLAS_PYTHON_INSTALL_DIR}/villas/node +) diff --git a/python/binding/module.cpp b/python/binding/module.cpp new file mode 100644 index 000000000..61167bd26 --- /dev/null +++ b/python/binding/module.cpp @@ -0,0 +1,305 @@ +/* Python bindings. + * + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace villas::node; + +// Intrusive holder over the reference count of a sample. +template class SampleRef { +public: + SampleRef() : ptr(nullptr) {} + explicit SampleRef(T *p) : ptr(p) { + if (ptr) + sample_incref(ptr); + } + SampleRef(const SampleRef &o) : SampleRef(o.ptr) {} + ~SampleRef() { + if (ptr) + sample_decref(ptr); + } + T *get() const { return ptr; } + +private: + T *ptr; +}; + +PYBIND11_DECLARE_HOLDER_TYPE(T, SampleRef, true) + +static SampleRef wrapNew(Sample *s) { + SampleRef ref(s); + sample_decref(s); + + return ref; +} + +static Node *makeNode(const std::string &config, const std::string &name) { + json_error_t err; + json_t *json = json_loads(config.c_str(), 0, &err); + if (!json) + throw py::value_error(err.text); + + uuid_t id; + uuid_clear(id); + + auto *n = NodeFactory::make(json, id, name); + if (!n) + throw py::value_error("failed to create node"); + + return n; +} + +// Lets a Python class override Hook::process() +class PyHook : public Hook { +public: + using Hook::Hook; + + Reason process(struct Sample *smp) override { + PYBIND11_OVERRIDE(Reason, Hook, process, smp); + } +}; + +PYBIND11_MODULE(_core, m) { + if (villas::node::memory::init(0)) + throw std::runtime_error("failed to initialize memory subsystem"); + + py::enum_(m, "State") + .value("INITIALIZED", State::INITIALIZED) + .value("PARSED", State::PARSED) + .value("CHECKED", State::CHECKED) + .value("PREPARED", State::PREPARED) + .value("STARTED", State::STARTED) + .value("STOPPED", State::STOPPED); + + py::class_>(m, "Sample", py::buffer_protocol()) + .def(py::init([](unsigned capacity) { + return wrapNew(sample_alloc_mem(capacity)); + }), + py::arg("capacity") = 64) + .def_readwrite("sequence", &Sample::sequence) + .def_readwrite("length", &Sample::length) + .def_readonly("capacity", &Sample::capacity) + .def_readwrite("flags", &Sample::flags) + .def_property_readonly("ts_origin", + [](const Sample &s) { + return std::make_pair(s.ts.origin.tv_sec, + s.ts.origin.tv_nsec); + }) + .def_buffer([](Sample &s) -> py::buffer_info { + return py::buffer_info( + reinterpret_cast(s.data), sizeof(double), + py::format_descriptor::format(), 1, + {static_cast(s.length)}, + {static_cast(sizeof(union SignalData))}); + }); + + py::class_(m, "Node") + .def(py::init([](const std::string &config, const std::string &name) { + return makeNode(config, name); + }), + py::arg("config"), py::arg("name") = "") + .def_property_readonly("name", &Node::getName) + .def_property_readonly("state", &Node::getState) + .def_property_readonly("details", &Node::getDetails) + .def("check", &Node::check) + .def("prepare", &Node::prepare) + .def("start", &Node::start, py::call_guard()) + .def("stop", &Node::stop, py::call_guard()) + .def( + "read", + [](Node &n, unsigned cnt, unsigned capacity) { + std::vector raw(cnt); + for (unsigned i = 0; i < cnt; i++) + raw[i] = sample_alloc_mem(capacity); + + int ret; + { + py::gil_scoped_release unlock; + ret = n.read(raw.data(), cnt); + } + + std::vector> out; + for (unsigned i = 0; i < cnt; i++) { + if (ret > 0 && i < (unsigned)ret) + out.push_back(wrapNew(raw[i])); + else + sample_decref(raw[i]); + } + + if (ret < 0) + throw std::runtime_error("node read failed"); + + return out; + }, + py::arg("cnt") = 1, py::arg("capacity") = 64) + .def( + "new_sample", + [](Node &n, unsigned capacity) { + auto *s = sample_alloc_mem(capacity); + + // Node::getOutputSignals() is null unless a path feeds this node + auto sigs = n.getOutputSignals(); + if (!sigs) + sigs = std::make_shared(capacity, SignalType::FLOAT); + + new (&s->signals) SignalList::Ptr(sigs); + + return wrapNew(s); + }, + py::arg("capacity") = 64) + .def( + "write", + [](Node &n, std::vector> smps) { + std::vector raw; + for (auto &s : smps) + raw.push_back(s.get()); + + py::gil_scoped_release unlock; + return n.write(raw.data(), raw.size()); + }, + py::arg("samples")) + .def_property_readonly("input_signals", + [](Node &n) { + std::vector names; + auto sigs = n.getInputSignals(false); + if (sigs) + for (const auto &s : *sigs) + names.push_back(s->name); + return names; + }) + .def("__repr__", [](Node &n) { + return ""; + }); + + py::class_(m, "NodeList") + .def(py::init<>()) + .def( + "append", [](NodeList &l, Node *n) { l.push_back(n); }, + py::keep_alive<1, 2>(), py::arg("node")) + .def("__len__", [](const NodeList &l) { return l.size(); }); + + py::enum_(m, "Reason") + .value("OK", Hook::Reason::OK) + .value("ERROR", Hook::Reason::ERROR) + .value("SKIP_SAMPLE", Hook::Reason::SKIP_SAMPLE) + .value("STOP_PROCESSING", Hook::Reason::STOP_PROCESSING); + + py::enum_(m, "HookFlags") + .value("BUILTIN", Hook::Flags::BUILTIN) + .value("PATH", Hook::Flags::PATH) + .value("NODE_READ", Hook::Flags::NODE_READ) + .value("NODE_WRITE", Hook::Flags::NODE_WRITE); + + py::class_ signalList(m, "SignalList"); + py::class_ path(m, "Path"); + py::class_ hook(m, "Hook"); + + signalList.def("__len__", [](const SignalList &l) { return l.size(); }) + .def("names", [](const SignalList &l) { + std::vector names; + for (const auto &s : l) + names.push_back(s->name); + + return names; + }); + + hook.def(py::init([](int flags, int priority, bool enabled) { + return new PyHook(nullptr, nullptr, flags, priority, enabled); + }), + py::arg("flags") = + (int)Hook::Flags::PATH | (int)Hook::Flags::BUILTIN, + py::arg("priority") = 100, py::arg("enabled") = true) + .def("process", &Hook::process, py::arg("sample")) + .def_property_readonly("signals", &Hook::getSignals); + + path.def(py::init<>()) + .def( + "parse", + [](Path &p, const std::string &config, NodeList &nodes) { + json_error_t err; + json_t *json = json_loads(config.c_str(), 0, &err); + if (!json) + throw py::value_error(err.text); + + uuid_t sn; + uuid_clear(sn); + + p.parse(json, nodes, sn); + }, + py::arg("config"), py::arg("nodes")) + .def("check", &Path::check) + .def("prepare", &Path::prepare, py::arg("nodes")) + .def("start", &Path::start, py::call_guard()) + .def("stop", &Path::stop, py::call_guard()) + .def( + "add_hook", [](Path &p, Hook::Ptr h) { p.hooks.push_back(h); }, + py::keep_alive<1, 2>(), py::arg("hook")) + .def_property_readonly("state", &Path::getState) + .def("__str__", &Path::toString); + + m.def("node_types", []() { + std::vector types; + for (auto *f : villas::plugin::registry->lookup()) + types.push_back(f->getName()); + + std::sort(types.begin(), types.end()); + + return types; + }); + + m.def("hook_types", []() { + std::vector types; + for (auto *f : villas::plugin::registry->lookup()) + types.push_back(f->getName()); + + std::sort(types.begin(), types.end()); + + return types; + }); + + py::class_(m, "SuperNode") + .def(py::init<>()) + .def( + "load", + [](SuperNode &sn, const std::string &path) { sn.parse(path); }, + py::arg("path")) + .def("check", &SuperNode::check) + .def("prepare", &SuperNode::prepare) + .def("start", &SuperNode::start, py::call_guard()) + .def("stop", &SuperNode::stop, py::call_guard()) + .def("node", &SuperNode::getNode, py::return_value_policy::reference, + py::arg("name")) + .def_property_readonly("node_names", + [](SuperNode &sn) { + std::vector names; + for (auto *n : sn.getNodes()) + names.push_back(n->getNameShort()); + + return names; + }) + .def_property_readonly("paths", [](SuperNode &sn) { + std::vector paths; + for (auto *p : sn.getPaths()) + paths.push_back(p->toString()); + + return paths; + }); +} diff --git a/python/examples/path_with_hook.py b/python/examples/path_with_hook.py new file mode 100644 index 000000000..ed6548bd1 --- /dev/null +++ b/python/examples/path_with_hook.py @@ -0,0 +1,87 @@ +""" +SPDX-FileCopyrightText: 2026 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import tempfile +import time + +import villas.node as vn + +OUTPUT = tempfile.mktemp(suffix=".dat") + +GENERATOR = { + "type": "signal.v2", + "rate": 10.0, + "realtime": True, + "limit": 20, + "in": {"signals": [{"name": "sine", "signal": "sine"}]}, +} + +SINK = { + "type": "file", + "uri": OUTPUT, + "format": "villas.human", +} + + +class Gain(vn.Hook): + """Signal processing in Python, called by the path thread for every sample""" + + def __init__(self, factor): + super().__init__(priority=50) + + self.factor = factor + self.calls = 0 + + def process(self, smp): + values = memoryview(smp) + for i in range(len(values)): + values[i] *= self.factor + + self.calls += 1 + + return vn.Reason.OK + + +def main(): + generator = vn.Node(json.dumps(GENERATOR), "generator") + sink = vn.Node(json.dumps(SINK), "sink") + + nodes = vn.NodeList() + nodes.append(generator) + nodes.append(sink) + + path = vn.Path() + path.parse(json.dumps({"in": "generator", "out": "sink"}), nodes) + + generator.check() + sink.check() + path.check() + + generator.prepare() + sink.prepare() + + gain = Gain(100.0) + path.add_hook(gain) + path.prepare(nodes) + + print(f"Starting {path}") + + generator.start() + sink.start() + path.start() + + time.sleep(2) + + path.stop() + generator.stop() + sink.stop() + + print(f"The hook processed {gain.calls} samples") + print(open(OUTPUT).read()) + + +if __name__ == "__main__": + main() diff --git a/python/pyproject.toml b/python/pyproject.toml index bd2c529b2..5fba8aec8 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,53 +1,6 @@ # SPDX-FileCopyrightText: 2023 OPAL-RT Germany GmbH # SPDX-License-Identifier: Apache-2.0 -[build-system] -requires = ['setuptools>=61.0'] -build-backend = 'setuptools.build_meta' - -[project] -name = 'villas-node' -version = "1.2.0" -description = 'Python support for the VILLASnode simulation-data gateway' -readme = 'README.md' -requires-python = '>=3.10' -keywords = ['simulation', 'power', 'system', 'real-time', 'villas'] -# license = 'Apache-2.0' -classifiers = [ - 'Development Status :: 4 - Beta', - 'Topic :: Scientific/Engineering', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3', -] -dependencies = [ - 'linuxfd==1.5; platform_system=="Linux"', - 'protobuf==6.32.1', - 'libconf==2.0.1', -] -optional-dependencies.dev = [ - 'black==25.11.0', - 'flake8==7.3.0', - 'mypy==1.19.0', - 'pytest==9.0.1', - 'types-protobuf==6.32.1.20251105', -] - -[project.scripts] -villas-conf2orchestra-ddf = "villas.node.opal_orchestra_ddf:main" - -[project.urls] -GitHub = 'https://github.com/VILLASframework/node' -Project = 'https://www.fein-aachen.org/en/projects/villas-node' -Homepage = 'https://villas.fein-aachen.org' - -[[project.authors]] -name = 'Steffen Vogel' -email = 'post@steffenvogel.de' - -[[project.authors]] -name = 'Philipp Jungkamp' -email = 'Philipp.Jungkamp@opal-rt.com' - [[tool.mypy.overrides]] module = ['google'] ignore_missing_imports = true @@ -56,10 +9,11 @@ ignore_missing_imports = true module = ['libconf'] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = ['villas.node._core'] +ignore_missing_imports = true + [tool.black] extend-exclude = '''.*(\.pyi|_pb2.py)$''' line-length = 90 - -[tool.setuptools.packages.find] -include = ["villas.node"] diff --git a/python/villas/node/__init__.py b/python/villas/node/__init__.py index 6b25e1978..622066a36 100644 --- a/python/villas/node/__init__.py +++ b/python/villas/node/__init__.py @@ -4,6 +4,39 @@ SPDX-License-Identifier: Apache-2.0 """ # noqa: E501 -from .node import Node +from .node import Node as Daemon -__all__ = ["Node"] +__all__ = ["Daemon"] + +try: + from ._core import ( + Hook, + HookFlags, + Node, + NodeList, + Path, + Reason, + Sample, + SignalList, + State, + SuperNode, + hook_types, + node_types, + ) +except ImportError: + pass +else: + __all__ += [ + "Hook", + "HookFlags", + "Node", + "NodeList", + "Path", + "Reason", + "Sample", + "SignalList", + "State", + "SuperNode", + "hook_types", + "node_types", + ] diff --git a/release-please-config.json b/release-please-config.json index 6d411deb4..73d52e7e2 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -8,7 +8,7 @@ "CMakeLists.txt", { "type": "toml", - "path": "python/pyproject.toml", + "path": "pyproject.toml", "jsonpath": "$.project.version" }, { diff --git a/tests/benchmarks/evaluate_logs.ipynb b/tests/benchmarks/evaluate_logs.ipynb index 3047336b1..4f176d0c8 100644 --- a/tests/benchmarks/evaluate_logs.ipynb +++ b/tests/benchmarks/evaluate_logs.ipynb @@ -761,7 +761,7 @@ " *ax.get_legend_handles_labels(),\n", " loc=\"upper left\",\n", " prop={\"family\": \"monospace\", \"size\": \"8\"},\n", - " ncol=settings[\"histograms\"][\"legend_columns\"]\n", + " ncol=settings[\"histograms\"][\"legend_columns\"],\n", " )\n", "\n", " figLegend.savefig(\"{}/legend_histogram.pdf\".format(rootdir), format=\"pdf\")" @@ -895,7 +895,7 @@ " loc=\"upper left\",\n", " labelspacing=1.2,\n", " prop={\"family\": \"monospace\", \"size\": \"8\"},\n", - " ncol=settings[\"median_plot\"][\"legend_columns\"]\n", + " ncol=settings[\"median_plot\"][\"legend_columns\"],\n", " )\n", "\n", " if settings[\"median_plot\"][\"print_missed_steps\"]:\n",