Merge branch 'sol1-468-review' into development

Author: bb-Ricardo

README.md

@@ -1,6 +1,9 @@
 
 # NetBox-Sync
 
+> [!CAUTION]
+> **Maintainer wanted - sunsetting this repository by 31.10.2025 [#474](https://github.com/bb-Ricardo/netbox-sync/issues/474)**
+
 This is a tool to sync data from different sources to a NetBox instance.
 
 Available source types:

module/netbox/__init__.py

@@ -17,6 +17,8 @@
     NBTenant,
     NBSite,
     NBSiteGroup,
+    NBRegion,
+    NBLocation,
     NBVRF,
     NBVLAN,
     NBVLANList,

module/netbox/object_classes.py

@@ -552,7 +552,6 @@ def update(self, data=None, read_from_netbox=False, source=None):
 
         parsed_data = dict()
         for key, value in data.items():
-
             if key not in self.data_model.keys():
                 log.error(f"Found undefined data model key '{key}' for object '{self.__class__.__name__}'")
                 continue
@@ -751,7 +750,7 @@ def update(self, data=None, read_from_netbox=False, source=None):
                 new_value_str = new_value_str.replace("\n", " ")
                 log.info(f"{self.name.capitalize()} '{display_name}' attribute '{key}' changed from "
                          f"'{current_value_str}' to '{new_value_str}'")
-
+            
             self.data[key] = new_value
             self.updated_items.append(key)
             data_updated = True
@@ -1273,7 +1272,6 @@ def get_site_name(self, data=None):
             if isinstance(this_site, dict):
                 return this_site.get("name")
 
-
 class NBObjectList(list):
     """
     Base class of listed NetBox objects. Extends list(). Currently used for tags and untagged VLANs
@@ -1424,39 +1422,39 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
 
-# class NBLocation(NetBoxObject):
-#     name = "location"
-#     api_path = "dcim/locations"
-#     object_type = "dcim.location"
-#     primary_key = "name"
-#     prune = False
-#     read_only = True
-#
-#     def __init__(self, *args, **kwargs):
-#         self.data_model = {
-#             "name": 100,
-#             "slug": 100,
-#             "site": NBSite,
-#             "tags": NBTagList
-#         }
-#         super().__init__(*args, **kwargs)
-#
-#
-# class NBRegion(NetBoxObject):
-#     name = "region"
-#     api_path = "dcim/regions"
-#     object_type = "dcim.region"
-#     primary_key = "name"
-#     prune = False
-#     read_only = True
-#
-#     def __init__(self, *args, **kwargs):
-#         self.data_model = {
-#             "name": 100,
-#             "slug": 100,
-#             "tags": NBTagList
-#         }
-#         super().__init__(*args, **kwargs)
+class NBLocation(NetBoxObject):
+    name = "location"
+    api_path = "dcim/locations"
+    object_type = "dcim.location"
+    primary_key = "name"
+    prune = False
+    read_only = True
+
+    def __init__(self, *args, **kwargs):
+        self.data_model = {
+            "name": 100,
+            "slug": 100,
+            "site": NBSite,
+            "tags": NBTagList
+        }
+        super().__init__(*args, **kwargs)
+
+
+class NBRegion(NetBoxObject):
+    name = "region"
+    api_path = "dcim/regions"
+    object_type = "dcim.region"
+    primary_key = "name"
+    prune = False
+    read_only = True
+
+    def __init__(self, *args, **kwargs):
+        self.data_model = {
+            "name": 100,
+            "slug": 100,
+            "tags": NBTagList
+        }
+        super().__init__(*args, **kwargs)
 
 
 class NBSite(NetBoxObject):
@@ -1868,14 +1866,14 @@ class NBCluster(NetBoxObject):
     api_path = "virtualization/clusters"
     object_type = "virtualization.cluster"
     primary_key = "name"
-    secondary_key = "site"
+    secondary_key = "scope_id"
     prune = False
-    # include_secondary_key_if_present = True
 
     def __init__(self, *args, **kwargs):
         self.mapping = NetBoxMappings()
+        # scope types allowed for clusters
         self.scopes = [
-            NBSite, NBSiteGroup
+            NBSite, NBSiteGroup, NBLocation, NBRegion
         ]
         self.data_model = {
             "name": 100,
@@ -1884,29 +1882,20 @@ def __init__(self, *args, **kwargs):
             "tenant": NBTenant,
             "group": NBClusterGroup,
             "scope_type": self.mapping.scopes_object_types(self.scopes),
-            # currently only site is supported as a scope
-            "scope_id": NBSite,
+            # supports scoped clusters
+            "scope_id": NetBoxObject,
+            # supports pre4.2.0 clusters with site
+            "site": NBSite,
             "tags": NBTagList
         }
         super().__init__(*args, **kwargs)
 
     def update(self, data=None, read_from_netbox=False, source=None):
 
-        # Add adaption for change in NetBox 4.2.0 Device model
-        if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"):
-            if data.get("site") is not None:
-                data["scope_id"] = data.get("site")
-                data["scope_type"] = "dcim.site"
-                del data["site"]
-
-            if data.get("scope_id") is not None:
-                data["scope_type"] = "dcim.site"
-
         super().update(data=data, read_from_netbox=read_from_netbox, source=source)
 
     def resolve_relations(self):
-
-        self.resolve_scoped_relations("scope_id", "scope_type")
+        log.debug2(f"Resolving relations for {self.name} '{self.get_display_name()}'")
         super().resolve_relations()
 
 

module/sources/vmware/config.py

@@ -143,6 +143,27 @@ def __init__(self):
                                              description="""Same as cluster site but on host level.
                                              If unset it will fall back to cluster_site_relation""",
                                              config_example="nyc02.* = New York, ffm01.* = Frankfurt"),
+                                ConfigOption("cluster_scope_type_relation", 
+                                             str,
+                                             description="""This option defines the scope type for a cluster.
+                                             The scope type can be 'dcim.site', 'dcim.sitegroup', 'dcim.location' or 'dcim.region'.
+                                             This is done with a comma separated key = value list.
+                                             Can be set to "<NONE>" to not assign a scope type.
+                                             Note: this does not remove scope types from existing clusters in NetBox.
+                                               key: defines a cluster name as regex
+                                               value: defines the NetBox scope type name (use quotes if name contains commas)
+                                             """,
+                                             config_example="Cluster_NYC = dcim.site, Cluster_FFM = dcim.sitegroup, Cluster_BER = dcim.location"),
+                                ConfigOption("cluster_scope_id_relation",
+                                             str,
+                                             description="""This option defines the scope id for a cluster.
+                                             The scope id is the NetBox ID of the scope type.
+                                             This is done with a comma separated key = value list.
+                                             To be used in combination with the 'cluster_scope_type_relation'.
+                                               key: defines a cluster name as regex
+                                               value: defines the NetBox scope id (use quotes if name contains commas)
+                                             """,
+                                             config_example="Cluster_NYC = 1, Cluster_FFM.* = 2, Cluster_BER = 7"),
                                 ConfigOption("cluster_tenant_relation",
                                              str,
                                              description="""\

module/sources/vmware/connection.py

@@ -62,6 +62,8 @@ class VMWareHandler(SourceBase):
         NBDeviceRole,
         NBSite,
         NBSiteGroup,
+        NBLocation,
+        NBRegion,
         NBCluster,
         NBDevice,
         NBVM,
@@ -474,13 +476,20 @@ def get_site_name(self, object_type, object_name, cluster_name=""):
 
         site_name = self.get_object_relation(object_name, relation_name)
 
-        if object_type == NBDevice and site_name is None:
+        # check if cluster is in a different site than the host and override the site name if so
+        if object_type == NBDevice:
             site_name = self.get_site_name(NBCluster, cluster_name)
             if site_name is not None:
-                log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'")
-
-        # set default site name
-        if site_name is None:
+                log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'. Overriding host site relation '{relation_name}'")
+            else:
+                site_name = self.get_object_relation(object_name, relation_name)
+                # set deault site name if no relation was found
+                if site_name is None:
+                    site_name = self.site_name
+                    log.debug2(f"No site relation for {type(object_name)}: '{object_name}' found, using default site '{site_name}'")
+
+        # set default site name for devices
+        if site_name is None and object_type == NBDevice:
             site_name = self.site_name
             log.debug(f"No site relation for '{object_name}' found, using default site '{site_name}'")
 
@@ -489,8 +498,95 @@ def get_site_name(self, object_type, object_name, cluster_name=""):
             site_name = None
             log.debug2(f"Site relation for '{object_name}' set to None")
 
+        log.debug2(f"Returning site name '{site_name}' for {object_type.name} '{object_name}'.")
+
         return site_name
+    
+    def get_scope_type(self, object_type, object_name):
+        """
+        Retrieve the scope_type for a NBCluster instance by object name or from the config option
+        cluster_scope_type_relation
+
+        Note: Only NBCluster is supported as the object_type.
+
+        Parameters
+        ----------
+        object_type: object type
+            The NetBox object type (must be NBCluster).
+        object_name: str
+            The name of the object to look up.
+
+        Returns
+        -------
+        str or None: scope type if one is found, otherwise None
+        """
+
+        # Validate object type
+        if object_type != NBCluster:
+            raise ValueError(f"Object type must be '{NBCluster.name}'.")
+        
+        # get scope type from relation config
+        relation_name = "cluster_scope_type_relation"
+        scope_type = self.get_object_relation(object_name, relation_name)
+        log.debug(f"Retrieved scope type '{scope_type}' for {object_type.name} '{object_name}' from relation '{relation_name}'.")
+        
+        # if the scope_type is a list, use the first element
+        if scope_type is not None and type(scope_type) is list:
+            scope_type_list = scope_type
+            scope_type = scope_type_list[0] if len(scope_type_list) > 0 else None
+            log.debug(f"Scope type for {object_type.name} '{object_name}' is a list, using first element: '{scope_type}'")
+
+        # if scope_type is not a str, return None
+        if type(scope_type) is not str:
+            log.debug(f"scope_type is type: {type(scope_type)}, not str")
+            return None
+
+        # set scope_type to None if it is configured as "<NONE>"
+        if scope_type == "<NONE>":
+            log.debug(f"Scope type for {object_type.name} '{object_name}' is set to None")
+            return None
+        
+        log.debug2(f"Returning scope type '{scope_type}' for {object_type.name} '{object_name}'.")
+        return scope_type
+
+    def get_scope_id(self, object_type, object_name):
+        """
+        Retrieve the scope_id for a NBCluster instance by object name or from the config option
+        cluster_scope_id_relation
+
+        Note: Only NBCluster is supported as the object_type.
 
+        Parameters
+        ----------
+        object_type: type
+            The NetBox object type (must be NBCluster).
+        object_name: str
+            The name of the object to look up.
+
+        Returns
+        -------
+        str or None: scope id if one is found, otherwise None
+        """
+        # Validate object type
+        if object_type != NBCluster:
+            raise ValueError(f"Object type must be '{NBCluster.name}'.")
+
+        # get scope id from relation config
+        relation_name = "cluster_scope_id_relation"
+        scope_id = self.get_object_relation(object_name, relation_name)
+
+        # return None if scope_id is None or not a string
+        if scope_id is None:
+            log.debug(f"No scope id found for {object_name}.") 
+            return None
+        if type(scope_id) is not str:
+            log.debug(f"scope_id is type: {type(scope_id)}, not str")
+            return None
+        
+        log.debug2(f"Retrieved scope id '{scope_id}' for {object_type.name} '{object_name}' from relation '{relation_name}'. End of method.")
+
+        return scope_id
+    
     def get_object_based_on_macs(self, object_type, mac_list=None):
         """
         Try to find a NetBox object based on list of MAC addresses.
@@ -1378,21 +1474,44 @@ def add_cluster(self, obj):
                                       self.settings.cluster_include_filter,
                                       self.settings.cluster_exclude_filter) is False:
             return
+        log.debug2(f"Cluster '{name}' passes include and exclude filters. Continuing.")
 
+        # get scope type and id, or site name
+        scope_type = self.get_scope_type(NBCluster, full_cluster_name)
+        if scope_type is None:
+            scope_type = self.get_scope_type(NBCluster, name)
+        
         site_name = self.get_site_name(NBCluster, full_cluster_name)
 
+        scope_id = self.get_scope_id(NBCluster, full_cluster_name)        
+        if scope_id is None:
+            scope_id = self.get_scope_id(NBCluster, name)
+        log.debug(f"Cluster '{full_cluster_name}' has scope id '{scope_id}' of type {type(scope_id)}.")
+
         data = {
             "name": name,
             "type": {"name": "VMware ESXi"},
             "group": group
         }
 
         if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"):
-            if site_name is not None:
-                data["scope_id"] = {"name": site_name}
+            # set the scope type and id if they are defined
+            if scope_type is not None:
+                data["scope_type"] = scope_type
+                data["scope_id"] = scope_id
+                log.debug(f"Cluster '{full_cluster_name}' (or {name}) has scope type '{scope_type}' "
+                          f"and scope id '{scope_id}'.")
+            elif site_name is not None:
                 data["scope_type"] = "dcim.site"
+                data["scope_id"] = {"name": site_name}
+            else:
+                log.debug(f"Cluster '{full_cluster_name}' has no scope type or scope id.")
         else:
-            data["site"] = {"name": site_name}
+            # set site_name in the pre-4.2.0 NetBox versions if one is found
+            if site_name is not None:
+                data["site"] = {"name": site_name}
+
+        log.debug(f"Cluster '{full_cluster_name}' (or {name}) has data items '{data.items()}'.")
 
         tenant_name = self.get_object_relation(full_cluster_name, "cluster_tenant_relation")
         if tenant_name is not None:
@@ -1411,11 +1530,12 @@ def add_cluster(self, obj):
             if grab(cluster_candidate, "data.name") != name:
                 continue
 
-            # try to find a cluster with matching site
-            if cluster_candidate.get_site_name() == site_name:
-                cluster_object = cluster_candidate
-                log.debug2("Found an existing cluster where 'name' and 'site' are matching")
-                break
+            if site_name is not None:
+                # try to find a cluster with matching site
+                if cluster_candidate.get_site_name() == site_name:
+                    cluster_object = cluster_candidate
+                    log.debug2("Found an existing cluster where 'name' and 'site' are matching")
+                    break
 
             if grab(cluster_candidate, "data.group") is not None and \
                     grab(cluster_candidate, "data.group.data.name") == group_name: