Skip to content

Nautobot NetBox Importer Package

nautobot_netbox_importer

App declaration for nautobot_netbox_importer.

NautobotNetboxImporterConfig

Bases: NautobotAppConfig

App configuration for the nautobot_netbox_importer app.

Source code in nautobot_netbox_importer/__init__.py
class NautobotNetboxImporterConfig(NautobotAppConfig):
    """App configuration for the nautobot_netbox_importer app."""

    name = "nautobot_netbox_importer"
    verbose_name = "Nautobot NetBox Importer"
    version = __version__
    author = "Network to Code, LLC"
    description = "Data importer from NetBox 3.x to Nautobot 2.x."
    base_url = "netbox-importer"
    required_settings = []
    min_version = "2.0.6"
    max_version = "2.9999"
    default_settings = {}
    caching_config = {}

base

Base types for the Nautobot Importer.

register_generator_setup(module)

Register adapter setup function.

This function must be called before the adapter is used and containing module can't import anything from Nautobot.

Source code in nautobot_netbox_importer/base.py
def register_generator_setup(module: str) -> None:
    """Register adapter setup function.

    This function must be called before the adapter is used and containing module can't import anything from Nautobot.
    """
    if module not in GENERATOR_SETUP_MODULES:
        GENERATOR_SETUP_MODULES.append(module)

command_utils

Utility functions and classes for nautobot_netbox_importer.

LogRenderer

Class for rendering structured logs to the console in a human-readable format.

Example

19:48:19 Apparent duplicate object encountered? duplicate_id: {'group': None, 'name': 'CR02.CHI_ORDMGMT', 'site': {'name': 'CHI01'}, 'vid': 1000} model: vlan pk_1: 3baf142d-dd90-4379-a048-3bbbcc9c799c pk_2: cba19791-4d59-4ddd-a5c9-d969ec3ed2ba

Source code in nautobot_netbox_importer/command_utils.py
class LogRenderer:  # pylint: disable=too-few-public-methods
    """Class for rendering structured logs to the console in a human-readable format.

    Example:
        19:48:19 Apparent duplicate object encountered?
          duplicate_id:
            {'group': None,
            'name': 'CR02.CHI_ORDMGMT',
            'site': {'name': 'CHI01'},
            'vid': 1000}
          model: vlan
          pk_1: 3baf142d-dd90-4379-a048-3bbbcc9c799c
          pk_2: cba19791-4d59-4ddd-a5c9-d969ec3ed2ba
    """

    def __call__(
        self,
        logger: structlog.types.WrappedLogger,
        name: str,
        event_dict: structlog.types.EventDict,
    ) -> str:
        """Render the given event_dict to a string."""
        sio = StringIO()

        timestamp = event_dict.pop("timestamp", None)
        if timestamp is not None:
            sio.write(f"{colorama.Style.DIM}{timestamp}{colorama.Style.RESET_ALL} ")

        level = event_dict.pop("level", None)
        if level is not None:
            if level in ("warning", "error", "critical"):
                sio.write(f"{colorama.Fore.RED}{level:<9}{colorama.Style.RESET_ALL}")
            else:
                sio.write(f"{level:<9}")

        event = event_dict.pop("event", None)
        sio.write(f"{colorama.Style.BRIGHT}{event}{colorama.Style.RESET_ALL}")

        for key, value in event_dict.items():
            if isinstance(value, dict):
                # We could use json.dumps() here instead of pprint.pformat,
                # but I find pprint to be a bit more compact while still readable.
                rendered_dict = pprint.pformat(value)
                if len(rendered_dict.splitlines()) > 50:  # noqa: PLR2004
                    rendered_dict = "\n".join(rendered_dict.splitlines()[:50]) + "\n...}"
                value = "\n" + textwrap.indent(rendered_dict, "    ")  # noqa: PLW2901
            sio.write(
                f"\n  {colorama.Fore.CYAN}{key}{colorama.Style.RESET_ALL}: "
                f"{colorama.Fore.MAGENTA}{value}{colorama.Style.RESET_ALL}"
            )

        return sio.getvalue()
__call__(logger, name, event_dict)

Render the given event_dict to a string.

Source code in nautobot_netbox_importer/command_utils.py
def __call__(
    self,
    logger: structlog.types.WrappedLogger,
    name: str,
    event_dict: structlog.types.EventDict,
) -> str:
    """Render the given event_dict to a string."""
    sio = StringIO()

    timestamp = event_dict.pop("timestamp", None)
    if timestamp is not None:
        sio.write(f"{colorama.Style.DIM}{timestamp}{colorama.Style.RESET_ALL} ")

    level = event_dict.pop("level", None)
    if level is not None:
        if level in ("warning", "error", "critical"):
            sio.write(f"{colorama.Fore.RED}{level:<9}{colorama.Style.RESET_ALL}")
        else:
            sio.write(f"{level:<9}")

    event = event_dict.pop("event", None)
    sio.write(f"{colorama.Style.BRIGHT}{event}{colorama.Style.RESET_ALL}")

    for key, value in event_dict.items():
        if isinstance(value, dict):
            # We could use json.dumps() here instead of pprint.pformat,
            # but I find pprint to be a bit more compact while still readable.
            rendered_dict = pprint.pformat(value)
            if len(rendered_dict.splitlines()) > 50:  # noqa: PLR2004
                rendered_dict = "\n".join(rendered_dict.splitlines()[:50]) + "\n...}"
            value = "\n" + textwrap.indent(rendered_dict, "    ")  # noqa: PLW2901
        sio.write(
            f"\n  {colorama.Fore.CYAN}{key}{colorama.Style.RESET_ALL}: "
            f"{colorama.Fore.MAGENTA}{value}{colorama.Style.RESET_ALL}"
        )

    return sio.getvalue()

enable_logging(verbosity=0, color=None)

Set up structlog (as used by DiffSync) to log messages for this command.

Source code in nautobot_netbox_importer/command_utils.py
def enable_logging(verbosity=0, color=None):
    """Set up structlog (as used by DiffSync) to log messages for this command."""
    if color is None:
        # Let colorama decide whether or not to strip out color codes
        colorama.init()
    else:
        # Force colors or non-colors, as specified
        colorama.init(strip=not color)

    structlog.configure(
        processors=[
            structlog.stdlib.add_log_level,
            structlog.processors.TimeStamper(fmt="%H:%M:%S"),
            LogRenderer(),
        ],
        context_class=dict,
        # Logging levels aren't very granular, so we adjust the log level based on *half* the verbosity level:
        # Verbosity     Logging level
        # 0             30 (WARNING)
        # 1-2           20 (INFO)
        # 3+            10 (DEBUG)
        wrapper_class=structlog.make_filtering_bound_logger(10 * (3 - ((verbosity + 1) // 2))),
        cache_logger_on_first_use=True,
    )

initialize_logger(options)

Initialize logger instance.

Source code in nautobot_netbox_importer/command_utils.py
def initialize_logger(options):
    """Initialize logger instance."""
    # Default of None means to use colorama's autodetection to determine whether or not to use color
    color = None
    if options.get("force_color"):
        color = True
    if options.get("no_color"):
        color = False

    enable_logging(verbosity=options["verbosity"], color=color)
    return structlog.get_logger(), color

diffsync

DiffSync adapter and model implementation for nautobot-netbox-importer.

adapters

Adapter classes for loading DiffSyncModels with data from NetBox or Nautobot.

NautobotAdapter

Bases: NautobotAdapter

DiffSync adapter for Nautobot.

Source code in nautobot_netbox_importer/diffsync/adapters/nautobot.py
class NautobotAdapter(_NautobotAdapter):
    """DiffSync adapter for Nautobot."""

    def __init__(self, *args, job=None, sync=None, **kwargs):
        """Initialize Nautobot.

        Args:
            *args (tuple): Arguments to be passed to the parent class.
            job (object, optional): Nautobot job. Defaults to None.
            sync (object, optional): Nautobot DiffSync. Defaults to None.
            **kwargs (dict): Additional arguments to be passed to the parent class.
        """
        super().__init__(*args, **kwargs)
        self.job = job
        self.sync = sync
__init__(*args, job=None, sync=None, **kwargs)

Initialize Nautobot.

Parameters:

Name Type Description Default
*args tuple

Arguments to be passed to the parent class.

()
job object

Nautobot job. Defaults to None.

None
sync object

Nautobot DiffSync. Defaults to None.

None
**kwargs dict

Additional arguments to be passed to the parent class.

{}
Source code in nautobot_netbox_importer/diffsync/adapters/nautobot.py
def __init__(self, *args, job=None, sync=None, **kwargs):
    """Initialize Nautobot.

    Args:
        *args (tuple): Arguments to be passed to the parent class.
        job (object, optional): Nautobot job. Defaults to None.
        sync (object, optional): Nautobot DiffSync. Defaults to None.
        **kwargs (dict): Additional arguments to be passed to the parent class.
    """
    super().__init__(*args, **kwargs)
    self.job = job
    self.sync = sync
NetBoxAdapter

Bases: SourceAdapter

NetBox Source Adapter.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
class NetBoxAdapter(SourceAdapter):
    """NetBox Source Adapter."""

    # pylint: disable=keyword-arg-before-vararg
    def __init__(self, input_ref: _FileRef, options: NetBoxImporterOptions, job=None, sync=None, *args, **kwargs):
        """Initialize NetBox Source Adapter."""
        super().__init__(
            name="NetBox",
            *args,
            get_source_data=_get_reader(input_ref),
            trace_issues=options.trace_issues,
            **kwargs,
        )
        self.job = job
        self.sync = sync

        self.options = options

        for name in GENERATOR_SETUP_MODULES:
            setup = __import__(name, fromlist=["setup"]).setup
            setup(self)

    def load(self) -> None:
        """Load data from NetBox."""
        self.import_data()
        if self.options.fix_powerfeed_locations:
            fix_power_feed_locations(self)
        if self.options.unrack_zero_uheight_devices:
            unrack_zero_uheight_devices(self)
        self.post_import()

    def import_to_nautobot(self) -> None:
        """Import a NetBox export file into Nautobot."""
        commited = False
        try:
            self._atomic_import()
            commited = True
        except _DryRunException:
            logger.warning("Dry-run mode, no data has been imported.")
        except _ImporterIssuesDetected:
            logger.warning("Importer issues detected, no data has been imported.")

        if commited and self.options.update_paths:
            logger.info("Updating paths ...")
            call_command("trace_paths", no_input=True)
            logger.info(" ... Updating paths completed.")

        if self.options.print_summary:
            self.summary.print()

    @atomic
    def _atomic_import(self) -> None:
        self.load()

        diff = self.nautobot.sync_from(self)
        self.summarize(diff.summary())

        if self.options.save_json_summary_path:
            self.summary.dump(self.options.save_json_summary_path, output_format="json")
        if self.options.save_text_summary_path:
            self.summary.dump(self.options.save_text_summary_path, output_format="text")

        has_issues = any(True for item in self.summary.nautobot if item.issues)
        if has_issues and not self.options.bypass_data_validation:
            raise _ImporterIssuesDetected("Importer issues detected, aborting the transaction.")

        if self.options.dry_run:
            raise _DryRunException("Aborting the transaction due to the dry-run mode.")
__init__(input_ref, options, job=None, sync=None, *args, **kwargs)

Initialize NetBox Source Adapter.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
def __init__(self, input_ref: _FileRef, options: NetBoxImporterOptions, job=None, sync=None, *args, **kwargs):
    """Initialize NetBox Source Adapter."""
    super().__init__(
        name="NetBox",
        *args,
        get_source_data=_get_reader(input_ref),
        trace_issues=options.trace_issues,
        **kwargs,
    )
    self.job = job
    self.sync = sync

    self.options = options

    for name in GENERATOR_SETUP_MODULES:
        setup = __import__(name, fromlist=["setup"]).setup
        setup(self)
import_to_nautobot()

Import a NetBox export file into Nautobot.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
def import_to_nautobot(self) -> None:
    """Import a NetBox export file into Nautobot."""
    commited = False
    try:
        self._atomic_import()
        commited = True
    except _DryRunException:
        logger.warning("Dry-run mode, no data has been imported.")
    except _ImporterIssuesDetected:
        logger.warning("Importer issues detected, no data has been imported.")

    if commited and self.options.update_paths:
        logger.info("Updating paths ...")
        call_command("trace_paths", no_input=True)
        logger.info(" ... Updating paths completed.")

    if self.options.print_summary:
        self.summary.print()
load()

Load data from NetBox.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
def load(self) -> None:
    """Load data from NetBox."""
    self.import_data()
    if self.options.fix_powerfeed_locations:
        fix_power_feed_locations(self)
    if self.options.unrack_zero_uheight_devices:
        unrack_zero_uheight_devices(self)
    self.post_import()
NetBoxImporterOptions

Bases: NamedTuple

NetBox importer options.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
class NetBoxImporterOptions(NamedTuple):
    """NetBox importer options."""

    dry_run: bool = True
    bypass_data_validation: bool = False
    print_summary: bool = False
    update_paths: bool = False
    fix_powerfeed_locations: bool = False
    sitegroup_parent_always_region: bool = False
    unrack_zero_uheight_devices: bool = True
    save_json_summary_path: str = ""
    save_text_summary_path: str = ""
    trace_issues: bool = False
nautobot

Nautobot Adapter for NetBox Importer.

NautobotAdapter

Bases: NautobotAdapter

DiffSync adapter for Nautobot.

Source code in nautobot_netbox_importer/diffsync/adapters/nautobot.py
class NautobotAdapter(_NautobotAdapter):
    """DiffSync adapter for Nautobot."""

    def __init__(self, *args, job=None, sync=None, **kwargs):
        """Initialize Nautobot.

        Args:
            *args (tuple): Arguments to be passed to the parent class.
            job (object, optional): Nautobot job. Defaults to None.
            sync (object, optional): Nautobot DiffSync. Defaults to None.
            **kwargs (dict): Additional arguments to be passed to the parent class.
        """
        super().__init__(*args, **kwargs)
        self.job = job
        self.sync = sync
__init__(*args, job=None, sync=None, **kwargs)

Initialize Nautobot.

Parameters:

Name Type Description Default
*args tuple

Arguments to be passed to the parent class.

()
job object

Nautobot job. Defaults to None.

None
sync object

Nautobot DiffSync. Defaults to None.

None
**kwargs dict

Additional arguments to be passed to the parent class.

{}
Source code in nautobot_netbox_importer/diffsync/adapters/nautobot.py
def __init__(self, *args, job=None, sync=None, **kwargs):
    """Initialize Nautobot.

    Args:
        *args (tuple): Arguments to be passed to the parent class.
        job (object, optional): Nautobot job. Defaults to None.
        sync (object, optional): Nautobot DiffSync. Defaults to None.
        **kwargs (dict): Additional arguments to be passed to the parent class.
    """
    super().__init__(*args, **kwargs)
    self.job = job
    self.sync = sync
netbox

NetBox to Nautobot Source Importer Definitions.

NetBoxAdapter

Bases: SourceAdapter

NetBox Source Adapter.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
class NetBoxAdapter(SourceAdapter):
    """NetBox Source Adapter."""

    # pylint: disable=keyword-arg-before-vararg
    def __init__(self, input_ref: _FileRef, options: NetBoxImporterOptions, job=None, sync=None, *args, **kwargs):
        """Initialize NetBox Source Adapter."""
        super().__init__(
            name="NetBox",
            *args,
            get_source_data=_get_reader(input_ref),
            trace_issues=options.trace_issues,
            **kwargs,
        )
        self.job = job
        self.sync = sync

        self.options = options

        for name in GENERATOR_SETUP_MODULES:
            setup = __import__(name, fromlist=["setup"]).setup
            setup(self)

    def load(self) -> None:
        """Load data from NetBox."""
        self.import_data()
        if self.options.fix_powerfeed_locations:
            fix_power_feed_locations(self)
        if self.options.unrack_zero_uheight_devices:
            unrack_zero_uheight_devices(self)
        self.post_import()

    def import_to_nautobot(self) -> None:
        """Import a NetBox export file into Nautobot."""
        commited = False
        try:
            self._atomic_import()
            commited = True
        except _DryRunException:
            logger.warning("Dry-run mode, no data has been imported.")
        except _ImporterIssuesDetected:
            logger.warning("Importer issues detected, no data has been imported.")

        if commited and self.options.update_paths:
            logger.info("Updating paths ...")
            call_command("trace_paths", no_input=True)
            logger.info(" ... Updating paths completed.")

        if self.options.print_summary:
            self.summary.print()

    @atomic
    def _atomic_import(self) -> None:
        self.load()

        diff = self.nautobot.sync_from(self)
        self.summarize(diff.summary())

        if self.options.save_json_summary_path:
            self.summary.dump(self.options.save_json_summary_path, output_format="json")
        if self.options.save_text_summary_path:
            self.summary.dump(self.options.save_text_summary_path, output_format="text")

        has_issues = any(True for item in self.summary.nautobot if item.issues)
        if has_issues and not self.options.bypass_data_validation:
            raise _ImporterIssuesDetected("Importer issues detected, aborting the transaction.")

        if self.options.dry_run:
            raise _DryRunException("Aborting the transaction due to the dry-run mode.")
__init__(input_ref, options, job=None, sync=None, *args, **kwargs)

Initialize NetBox Source Adapter.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
def __init__(self, input_ref: _FileRef, options: NetBoxImporterOptions, job=None, sync=None, *args, **kwargs):
    """Initialize NetBox Source Adapter."""
    super().__init__(
        name="NetBox",
        *args,
        get_source_data=_get_reader(input_ref),
        trace_issues=options.trace_issues,
        **kwargs,
    )
    self.job = job
    self.sync = sync

    self.options = options

    for name in GENERATOR_SETUP_MODULES:
        setup = __import__(name, fromlist=["setup"]).setup
        setup(self)
import_to_nautobot()

Import a NetBox export file into Nautobot.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
def import_to_nautobot(self) -> None:
    """Import a NetBox export file into Nautobot."""
    commited = False
    try:
        self._atomic_import()
        commited = True
    except _DryRunException:
        logger.warning("Dry-run mode, no data has been imported.")
    except _ImporterIssuesDetected:
        logger.warning("Importer issues detected, no data has been imported.")

    if commited and self.options.update_paths:
        logger.info("Updating paths ...")
        call_command("trace_paths", no_input=True)
        logger.info(" ... Updating paths completed.")

    if self.options.print_summary:
        self.summary.print()
load()

Load data from NetBox.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
def load(self) -> None:
    """Load data from NetBox."""
    self.import_data()
    if self.options.fix_powerfeed_locations:
        fix_power_feed_locations(self)
    if self.options.unrack_zero_uheight_devices:
        unrack_zero_uheight_devices(self)
    self.post_import()
NetBoxImporterOptions

Bases: NamedTuple

NetBox importer options.

Source code in nautobot_netbox_importer/diffsync/adapters/netbox.py
class NetBoxImporterOptions(NamedTuple):
    """NetBox importer options."""

    dry_run: bool = True
    bypass_data_validation: bool = False
    print_summary: bool = False
    update_paths: bool = False
    fix_powerfeed_locations: bool = False
    sitegroup_parent_always_region: bool = False
    unrack_zero_uheight_devices: bool = True
    save_json_summary_path: str = ""
    save_text_summary_path: str = ""
    trace_issues: bool = False

models

NetBox Importer DiffSync Models.

base

NetBox to Nautobot Base Models Mapping.

setup(adapter)

Map NetBox base models to Nautobot.

Source code in nautobot_netbox_importer/diffsync/models/base.py
def setup(adapter: SourceAdapter) -> None:
    """Map NetBox base models to Nautobot."""
    adapter.disable_model("sessions.session", "Nautobot has own sessions, sessions should never cross apps.")
    adapter.disable_model("admin.logentry", "Not directly used in Nautobot.")
    adapter.disable_model("users.userconfig", "May not have a 1 to 1 translation to Nautobot.")
    adapter.disable_model("auth.permission", "Handled via a Nautobot model and may not be a 1 to 1.")

    _setup_content_types(adapter)

    adapter.configure_model(
        "extras.Status",
        identifiers=["name"],
        default_reference={
            "name": "Unknown",
        },
    )
    adapter.configure_model("extras.role")
    adapter.configure_model(
        "extras.tag",
        fields={
            "object_types": "content_types",
        },
    )
    adapter.configure_model(
        "extras.TaggedItem",
        fields={
            "object_id": _define_tagged_object,
        },
    )
    adapter.configure_model(
        "extras.ConfigContext",
        fields={
            "locations": define_locations,
            "roles": fields.relation("dcim.DeviceRole"),
        },
    )
    adapter.configure_model(
        # pylint: disable=hard-coded-auth-user
        "auth.User",
        nautobot_content_type="users.User",
        identifiers=["username"],
        fields={
            "last_login": fields.disable("Should not be attempted to migrate"),
            "password": fields.disable("Should not be attempted to migrate"),
            "user_permissions": fields.disable("Permissions import is not implemented yet"),
        },
    )
    adapter.configure_model(
        "auth.Group",
        identifiers=["name"],
        fields={
            "permissions": fields.disable("Permissions import is not implemented yet"),
        },
    )
    adapter.configure_model(
        "tenancy.Tenant",
        fields={
            "group": "tenant_group",
        },
    )
    adapter.configure_model(
        "extras.JournalEntry",
        nautobot_content_type="extras.Note",
    )
circuits

NetBox to Nautobot Circuits Models Mapping.

setup(adapter)

Map NetBox circuits models to Nautobot.

Source code in nautobot_netbox_importer/diffsync/models/circuits.py
def setup(adapter: SourceAdapter) -> None:
    """Map NetBox circuits models to Nautobot."""
    adapter.configure_model(
        "circuits.circuit",
        fields={
            "type": "circuit_type",
            "termination_a": "circuit_termination_a",
            "termination_z": "circuit_termination_z",
        },
    )
    adapter.configure_model(
        "circuits.circuittermination",
        fields={
            "location": define_location,
        },
    )
custom_fields

NetBox to Nautobot Custom Fields Models Mapping.

setup(adapter)

Map NetBox custom fields to Nautobot.

Source code in nautobot_netbox_importer/diffsync/models/custom_fields.py
def setup(adapter: SourceAdapter) -> None:
    """Map NetBox custom fields to Nautobot."""
    choice_sets = {}

    def create_choice_set(source: RecordData, importer_pass: ImporterPass) -> PreImportResult:
        if importer_pass == ImporterPass.DEFINE_STRUCTURE:
            choice_sets[source.get("id")] = [
                *_convert_choices(source.get("base_choices")),
                *_convert_choices(source.get("extra_choices")),
            ]

        return PreImportResult.USE_RECORD

    def define_choice_set(field: SourceField) -> None:
        def choices_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            choice_set = source.get(field.name, None)
            if choice_set in EMPTY_VALUES:
                return

            choices = choice_sets.get(choice_set, None)
            if not choices:
                raise ValueError(f"Choice set {choice_set} not found")

            create_choices(choices, getattr(target, "id"))

        field.set_importer(choices_importer, nautobot_name=None)

    def define_choices(field: SourceField) -> None:
        def choices_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            choices = _convert_choices(source.get(field.name, None))
            if choices in EMPTY_VALUES:
                return

            if not isinstance(choices, list):
                raise ValueError(f"Choices must be a list of strings, got {type(choices)}")

            create_choices(choices, getattr(target, "id"))

        field.set_importer(choices_importer, nautobot_name=None)

    def create_choices(choices: list, custom_field_uid: Uid) -> None:
        for choice in choices:
            choices_wrapper.import_record(
                {
                    "id": choice,
                    "custom_field": custom_field_uid,
                    "value": choice,
                },
            )

    # Defined in NetBox but not in Nautobot
    adapter.configure_model(
        "extras.CustomFieldChoiceSet",
        pre_import=create_choice_set,
    )

    adapter.configure_model(
        "extras.CustomField",
        fields={
            "name": "key",
            "label": fields.default("Empty Label"),
            "type": _define_custom_field_type,
            # NetBox<3.6
            "choices": define_choices,
            # NetBox>=3.6
            "choice_set": define_choice_set,
        },
    )

    choices_wrapper = adapter.configure_model(
        "extras.CustomFieldChoice",
        fields={
            "custom_field": "custom_field",
            "value": "value",
        },
    )
dcim

NetBox to Nautobot DCIM Models Mapping.

fix_power_feed_locations(adapter)

Fix panel location to match rack location based on powerfeed.

Source code in nautobot_netbox_importer/diffsync/models/dcim.py
def fix_power_feed_locations(adapter: SourceAdapter) -> None:
    """Fix panel location to match rack location based on powerfeed."""
    region_wrapper = adapter.wrappers["dcim.region"]
    site_wrapper = adapter.wrappers["dcim.site"]
    location_wrapper = adapter.wrappers["dcim.location"]
    rack_wrapper = adapter.wrappers["dcim.rack"]
    panel_wrapper = adapter.wrappers["dcim.powerpanel"]

    diffsync_class = adapter.wrappers["dcim.powerfeed"].nautobot.diffsync_class

    for item in adapter.get_all(diffsync_class):
        rack_id = getattr(item, "rack_id", None)
        panel_id = getattr(item, "power_panel_id", None)
        if not (rack_id and panel_id):
            continue

        rack = rack_wrapper.get_or_create(rack_id)
        panel = panel_wrapper.get_or_create(panel_id)

        rack_location_uid = getattr(rack, "location_id", None)
        panel_location_uid = getattr(panel, "location_id", None)
        if rack_location_uid == panel_location_uid:
            continue

        if rack_location_uid:
            location_uid = rack_location_uid
            target = panel
            target_wrapper = panel_wrapper
        else:
            location_uid = panel_location_uid
            target = rack
            target_wrapper = rack_wrapper

        if not isinstance(location_uid, UUID):
            raise TypeError(f"Location UID must be UUID, got {type(location_uid)}")

        target.location_id = location_uid
        adapter.update(target)

        # Need to update references, to properly update `content_types` fields
        # References can be counted and removed, if needed
        if location_uid in region_wrapper.references:
            target_wrapper.add_reference(region_wrapper, location_uid)
        elif location_uid in site_wrapper.references:
            target_wrapper.add_reference(site_wrapper, location_uid)
        elif location_uid in location_wrapper.references:
            target_wrapper.add_reference(location_wrapper, location_uid)
        else:
            raise ValueError(f"Unknown location type {location_uid}")
setup(adapter)

Map NetBox DCIM models to Nautobot.

Source code in nautobot_netbox_importer/diffsync/models/dcim.py
def setup(adapter: SourceAdapter) -> None:
    """Map NetBox DCIM models to Nautobot."""
    adapter.disable_model("dcim.cablepath", "Recreated in Nautobot on signal when circuit termination is created")
    adapter.configure_model(
        "dcim.rackreservation",
        fields={
            # Set the definition of the `units` field
            "units": _define_units,
        },
    )
    adapter.configure_model(
        "dcim.rack",
        fields={
            "location": define_location,
            "role": fields.role(adapter, "dcim.rackrole"),
        },
    )
    adapter.configure_model("dcim.cable")
    adapter.configure_model(
        "dcim.cabletermination",
        extend_content_type="dcim.cable",
        pre_import=_pre_import_cable_termination,
    )
    adapter.configure_model(
        "dcim.interface",
        fields={
            "parent": "parent_interface",
        },
    )
    manufacturer = adapter.configure_model(
        "dcim.manufacturer",
        default_reference={
            "id": "Unknown",
            "name": "Unknown",
        },
    )
    adapter.configure_model(
        "dcim.devicetype",
        fields={
            "front_image": fields.disable("Import does not contain images"),
            "rear_image": fields.disable("Import does not contain images"),
        },
        default_reference={
            "id": "Unknown",
            "manufacturer": manufacturer.get_default_reference_uid(),
            "model": "Unknown",
        },
    )
    adapter.configure_model(
        "dcim.device",
        fields={
            "location": define_location,
            "device_role": fields.role(adapter, "dcim.devicerole"),
            "role": fields.role(adapter, "dcim.devicerole"),
        },
    )
    adapter.configure_model(
        "dcim.powerpanel",
        fields={
            "location": define_location,
        },
    )
    adapter.configure_model(
        "dcim.frontporttemplate",
        fields={
            "rear_port": "rear_port_template",
        },
    )
    adapter.configure_model(
        "dcim.poweroutlettemplate",
        fields={
            "power_port": "power_port_template",
        },
    )
unrack_zero_uheight_devices(adapter)

Unrack devices with 0U height.

Source code in nautobot_netbox_importer/diffsync/models/dcim.py
def unrack_zero_uheight_devices(adapter: SourceAdapter) -> None:
    """Unrack devices with 0U height."""
    device_wrapper = adapter.wrappers["dcim.device"]
    device_type_wrapper = adapter.wrappers["dcim.devicetype"]

    # Find all device types with 0U height
    device_type_ids = set(
        getattr(item, "id")
        for item in adapter.get_all(device_type_wrapper.nautobot.diffsync_class)
        if getattr(item, "u_height", 0) == 0
    )

    if not device_type_ids:
        return

    # Update all devices with matching device type, clean `position` field.
    position = device_wrapper.fields["position"]
    for item in adapter.get_all(device_wrapper.nautobot.diffsync_class):
        if getattr(item, "position", None) and getattr(item, "device_type_id") in device_type_ids:
            position.set_nautobot_value(item, None)
            adapter.update(item)
            position.add_issue("Unracked", "Device unracked due to 0U height", item)
ipam

NetBox to Nautobot IPAM Models Mapping.

setup(adapter)

Map NetBox IPAM models to Nautobot.

Source code in nautobot_netbox_importer/diffsync/models/ipam.py
def setup(adapter: SourceAdapter) -> None:
    """Map NetBox IPAM models to Nautobot."""
    ipaddress = adapter.configure_model(
        "ipam.ipaddress",
        fields={
            "role": fields.role(adapter, "ipam.role"),
        },
    )
    ipaddress.nautobot.set_instance_defaults(namespace=get_default_namespace())
    adapter.configure_model(
        "ipam.prefix",
        fields={
            "location": define_location,
            "role": fields.role(adapter, "ipam.role"),
        },
    )
    adapter.configure_model(
        "ipam.aggregate",
        nautobot_content_type="ipam.prefix",
    )
    adapter.configure_model(
        "ipam.vlan",
        fields={
            "group": "vlan_group",
            "location": define_location,
            "role": fields.role(adapter, "ipam.role"),
        },
    )
    adapter.configure_model(
        "ipam.FHRPGroup",
        nautobot_content_type="dcim.InterfaceRedundancyGroup",
        fields={
            "protocol": fields.fallback(callback=_fhrp_protocol_fallback),
        },
    )
locations

NetBox Specific Locations handling.

define_location(field)

Define location field for NetBox importer.

Source code in nautobot_netbox_importer/diffsync/models/locations.py
def define_location(field: SourceField) -> None:
    """Define location field for NetBox importer."""
    wrapper = field.wrapper

    location_wrapper = wrapper.adapter.wrappers["dcim.location"]
    site_wrapper = wrapper.adapter.wrappers["dcim.site"]
    region_wrapper = wrapper.adapter.wrappers["dcim.region"]

    # Nautobot v2.2.0 uses a ManyToManyField `locations` field instead of a ForeignKey `location`
    locations = wrapper.nautobot.add_field("locations").internal_type == InternalFieldType.MANY_TO_MANY_FIELD

    def location_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        location = source.get(field.name, None)
        site = source.get("site", None)
        region = source.get("region", None)

        # Location is the most specific, then site, the last is region
        if location:
            result = location_wrapper.get_pk_from_uid(location)
            wrapper.add_reference(location_wrapper, result)
        elif site:
            result = site_wrapper.get_pk_from_uid(site)
            wrapper.add_reference(site_wrapper, result)
        elif region:
            result = region_wrapper.get_pk_from_uid(region)
            wrapper.add_reference(region_wrapper, result)
        else:
            return

        field.set_nautobot_value(target, set([result]) if locations else result)

    field.set_importer(location_importer, "locations" if locations else "location")
    field.handle_sibling("site", field.nautobot.name)
    field.handle_sibling("region", field.nautobot.name)
define_locations(field)

Define locations field for NetBox importer.

Source code in nautobot_netbox_importer/diffsync/models/locations.py
def define_locations(field: SourceField) -> None:
    """Define locations field for NetBox importer."""
    wrapper = field.wrapper

    location_wrapper = wrapper.adapter.wrappers["dcim.location"]
    site_wrapper = wrapper.adapter.wrappers["dcim.site"]
    region_wrapper = wrapper.adapter.wrappers["dcim.region"]

    def location_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        locations = source.get(field.name, None)
        sites = source.get("sites", None)
        regions = source.get("regions", None)

        result = set()

        if locations:
            for location in locations:
                location_uid = location_wrapper.get_pk_from_uid(location)
                wrapper.add_reference(location_wrapper, location_uid)
                result.add(location_uid)

        if sites:
            for site in sites:
                location_uid = site_wrapper.get_pk_from_uid(site)
                wrapper.add_reference(location_wrapper, location_uid)
                result.add(location_uid)

        if regions:
            for region in regions:
                location_uid = region_wrapper.get_pk_from_uid(region)
                wrapper.add_reference(location_wrapper, location_uid)
                result.add(location_uid)

        field.set_nautobot_value(target, result)

    field.set_importer(location_importer, "locations")
    field.handle_sibling("sites", field.nautobot.name)
    field.handle_sibling("regions", field.nautobot.name)
setup(adapter)

Setup locations for NetBox importer.

Source code in nautobot_netbox_importer/diffsync/models/locations.py
def setup(adapter: SourceAdapter) -> None:
    """Setup locations for NetBox importer."""

    def forward_references(wrapper: SourceModelWrapper, references: SourceReferences) -> None:
        """Forward references to Location, Site and Region instance to their LocationType to fill `content_types`."""
        for uid, wrappers in references.items():
            instance = wrapper.get_or_create(uid, fail_missing=True)
            location_type_uid = getattr(instance, "location_type_id")
            for item in wrappers:
                item.add_reference(location_type_wrapper, location_type_uid)

    options = getattr(adapter, "options", {})
    sitegroup_parent_always_region = getattr(options, "sitegroup_parent_always_region", False)

    location_type_wrapper = adapter.configure_model("dcim.LocationType")

    region_type_uid = location_type_wrapper.cache_record(
        {
            "id": "Region",
            "name": "Region",
            "nestable": True,
        }
    )
    site_type_uid = location_type_wrapper.cache_record(
        {
            "id": "Site",
            "name": "Site",
            "nestable": False,
            "parent": region_type_uid,
        }
    )
    location_type_uid = location_type_wrapper.cache_record(
        {
            "id": "Location",
            "name": "Location",
            "nestable": True,
            "parent": site_type_uid,
        }
    )

    adapter.configure_model(
        "dcim.SiteGroup",
        nautobot_content_type="dcim.LocationType",
        fields={
            "parent": fields.constant(region_type_uid) if sitegroup_parent_always_region else "parent",
            "nestable": fields.constant(True),
        },
    )

    adapter.configure_model(
        "dcim.Region",
        nautobot_content_type="dcim.Location",
        forward_references=forward_references,
        fields={
            "location_type": fields.constant(region_type_uid),
        },
    )

    adapter.configure_model(
        "dcim.Site",
        nautobot_content_type="dcim.Location",
        forward_references=forward_references,
        fields={
            "region": fields.relation("dcim.Region", "parent"),
            "group": _define_site_group,
        },
    )

    adapter.configure_model(
        "dcim.Location",
        forward_references=forward_references,
        fields={
            "location_type": fields.constant(location_type_uid),
            "parent": _define_location_parent,
        },
    )
object_change

NetBox to Nautobot Object Change Model Mapping.

setup(adapter)

Map NetBox object change to Nautobot.

Source code in nautobot_netbox_importer/diffsync/models/object_change.py
def setup(adapter: SourceAdapter) -> None:
    """Map NetBox object change to Nautobot."""

    def skip_disabled_object_types(source: RecordData, importer_pass: ImporterPass) -> PreImportResult:
        """Disabled object types are not in Nautobot and should be skipped."""
        if importer_pass != ImporterPass.IMPORT_DATA:
            return PreImportResult.USE_RECORD
        object_type = source.get("changed_object_type", None)
        wrapper = adapter.get_or_create_wrapper(object_type)
        return PreImportResult.SKIP_RECORD if wrapper.disable_reason else PreImportResult.USE_RECORD

    adapter.configure_model(
        "extras.ObjectChange",
        pre_import=skip_disabled_object_types,
        disable_related_reference=True,
        fields={
            "postchange_data": "object_data",
            # TBD: This should be defined on Nautobot side
            "time": fields.force(),
        },
    )
virtualization

NetBox to Nautobot Virtualization Models Mapping.

setup(adapter)

Map NetBox virtualization models to Nautobot.

Source code in nautobot_netbox_importer/diffsync/models/virtualization.py
def setup(adapter: SourceAdapter) -> None:
    """Map NetBox virtualization models to Nautobot."""
    adapter.configure_model(
        "virtualization.cluster",
        fields={
            "type": "cluster_type",
            "group": "cluster_group",
            "location": define_location,
        },
    )
    adapter.configure_model(
        "virtualization.virtualmachine",
        fields={
            "role": fields.role(adapter, "dcim.devicerole"),
        },
    )
    adapter.configure_model(
        "virtualization.vminterface",
        fields={
            "parent": "parent_interface",
        },
    )

generator

Generic Nautobot Import Library using DiffSync.

DiffSyncBaseModel

Bases: DiffSyncModel

Base class for all DiffSync models.

Source code in nautobot_netbox_importer/generator/nautobot.py
class DiffSyncBaseModel(DiffSyncModel):
    """Base class for all DiffSync models."""

    _wrapper: NautobotModelWrapper

    @classmethod
    def create(cls, diffsync: DiffSync, ids: dict, attrs: dict) -> Optional["DiffSyncBaseModel"]:
        """Create this model instance, both in Nautobot and in DiffSync."""
        wrapper = cls._wrapper

        instance = None
        try:
            instance = wrapper.model(**wrapper.constructor_kwargs, **ids)
            result = super().create(diffsync, ids, attrs)
        # pylint: disable=broad-exception-caught
        except Exception as error:
            wrapper.add_issue(
                "CreateFailed",
                uid=ids.get(wrapper.pk_field.name, ""),
                nautobot_instance=instance,
                data=attrs,
                error=error,
            )
            return None

        if wrapper.save_nautobot_instance(instance, attrs):
            wrapper.stats.created += 1
            return result

        return None

    def update(self, attrs: dict) -> Optional["DiffSyncBaseModel"]:
        """Update this model instance, both in Nautobot and in DiffSync."""
        wrapper = self._wrapper
        uid = getattr(self, wrapper.pk_field.name, None)
        if not uid:
            raise NotImplementedError("Cannot update model without pk")

        try:
            super().update(attrs)
        # pylint: disable=broad-exception-caught
        except Exception as error:
            wrapper.add_issue(
                "UpdateFailed",
                uid=uid,
                diffsync_instance=self,
                data=attrs,
                error=error,
            )
            return None

        model = wrapper.model
        filter_kwargs = {wrapper.pk_field.name: uid}
        instance = model.objects.get(**filter_kwargs)
        if wrapper.save_nautobot_instance(instance, attrs):
            wrapper.stats.updated += 1
            return self

        return None
create(diffsync, ids, attrs) classmethod

Create this model instance, both in Nautobot and in DiffSync.

Source code in nautobot_netbox_importer/generator/nautobot.py
@classmethod
def create(cls, diffsync: DiffSync, ids: dict, attrs: dict) -> Optional["DiffSyncBaseModel"]:
    """Create this model instance, both in Nautobot and in DiffSync."""
    wrapper = cls._wrapper

    instance = None
    try:
        instance = wrapper.model(**wrapper.constructor_kwargs, **ids)
        result = super().create(diffsync, ids, attrs)
    # pylint: disable=broad-exception-caught
    except Exception as error:
        wrapper.add_issue(
            "CreateFailed",
            uid=ids.get(wrapper.pk_field.name, ""),
            nautobot_instance=instance,
            data=attrs,
            error=error,
        )
        return None

    if wrapper.save_nautobot_instance(instance, attrs):
        wrapper.stats.created += 1
        return result

    return None
update(attrs)

Update this model instance, both in Nautobot and in DiffSync.

Source code in nautobot_netbox_importer/generator/nautobot.py
def update(self, attrs: dict) -> Optional["DiffSyncBaseModel"]:
    """Update this model instance, both in Nautobot and in DiffSync."""
    wrapper = self._wrapper
    uid = getattr(self, wrapper.pk_field.name, None)
    if not uid:
        raise NotImplementedError("Cannot update model without pk")

    try:
        super().update(attrs)
    # pylint: disable=broad-exception-caught
    except Exception as error:
        wrapper.add_issue(
            "UpdateFailed",
            uid=uid,
            diffsync_instance=self,
            data=attrs,
            error=error,
        )
        return None

    model = wrapper.model
    filter_kwargs = {wrapper.pk_field.name: uid}
    instance = model.objects.get(**filter_kwargs)
    if wrapper.save_nautobot_instance(instance, attrs):
        wrapper.stats.updated += 1
        return self

    return None

ImporterPass

Bases: Enum

Importer Pass.

Source code in nautobot_netbox_importer/generator/source.py
class ImporterPass(Enum):
    """Importer Pass."""

    DEFINE_STRUCTURE = 1
    IMPORT_DATA = 2

InternalFieldType

Bases: Enum

Internal field types.

Source code in nautobot_netbox_importer/generator/base.py
class InternalFieldType(Enum):
    """Internal field types."""

    AUTO_FIELD = "AutoField"
    BIG_AUTO_FIELD = "BigAutoField"
    BIG_INTEGER_FIELD = "BigIntegerField"
    BINARY_FIELD = "BinaryField"
    BOOLEAN_FIELD = "BooleanField"
    CHAR_FIELD = "CharField"
    CUSTOM_FIELD_DATA = "CustomFieldData"
    DATE_FIELD = "DateField"
    DATE_TIME_FIELD = "DateTimeField"
    DECIMAL_FIELD = "DecimalField"
    FOREIGN_KEY = "ForeignKey"
    FOREIGN_KEY_WITH_AUTO_RELATED_NAME = "ForeignKeyWithAutoRelatedName"
    INTEGER_FIELD = "IntegerField"
    JSON_FIELD = "JSONField"
    MANY_TO_MANY_FIELD = "ManyToManyField"
    NOT_FOUND = "NotFound"
    ONE_TO_ONE_FIELD = "OneToOneField"
    POSITIVE_INTEGER_FIELD = "PositiveIntegerField"
    POSITIVE_SMALL_INTEGER_FIELD = "PositiveSmallIntegerField"
    PRIVATE_PROPERTY = "PrivateProperty"
    PROPERTY = "Property"
    READ_ONLY_PROPERTY = "ReadOnlyProperty"
    ROLE_FIELD = "RoleField"
    SLUG_FIELD = "SlugField"
    SMALL_INTEGER_FIELD = "SmallIntegerField"
    STATUS_FIELD = "StatusField"
    TEXT_FIELD = "TextField"
    TREE_NODE_FOREIGN_KEY = "TreeNodeForeignKey"
    UUID_FIELD = "UUIDField"

InvalidChoiceValueIssue

Bases: SourceFieldImporterIssue

Raised when an invalid choice value is encountered.

Source code in nautobot_netbox_importer/generator/source.py
class InvalidChoiceValueIssue(SourceFieldImporterIssue):
    """Raised when an invalid choice value is encountered."""

    def __init__(self, field: "SourceField", value: Any, replacement: Any = NOTHING):
        """Initialize the exception."""
        message = f"Invalid choice value: `{value}`"
        if replacement is not NOTHING:
            message += f", replaced with `{replacement}`"
        super().__init__(message, field)
__init__(field, value, replacement=NOTHING)

Initialize the exception.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(self, field: "SourceField", value: Any, replacement: Any = NOTHING):
    """Initialize the exception."""
    message = f"Invalid choice value: `{value}`"
    if replacement is not NOTHING:
        message += f", replaced with `{replacement}`"
    super().__init__(message, field)

NautobotAdapter

Bases: BaseAdapter

Nautobot DiffSync Adapter.

Source code in nautobot_netbox_importer/generator/nautobot.py
class NautobotAdapter(BaseAdapter):
    """Nautobot DiffSync Adapter."""

    def __init__(self, *args, **kwargs):
        """Initialize the adapter."""
        super().__init__("Nautobot", *args, **kwargs)
        self.wrappers: Dict[ContentTypeStr, NautobotModelWrapper] = {}
        self.trace_issues = False

    def get_or_create_wrapper(self, content_type: ContentTypeStr) -> "NautobotModelWrapper":
        """Get or create a Nautobot model wrapper."""
        if content_type in self.wrappers:
            return self.wrappers[content_type]

        return NautobotModelWrapper(self, content_type)
__init__(*args, **kwargs)

Initialize the adapter.

Source code in nautobot_netbox_importer/generator/nautobot.py
def __init__(self, *args, **kwargs):
    """Initialize the adapter."""
    super().__init__("Nautobot", *args, **kwargs)
    self.wrappers: Dict[ContentTypeStr, NautobotModelWrapper] = {}
    self.trace_issues = False
get_or_create_wrapper(content_type)

Get or create a Nautobot model wrapper.

Source code in nautobot_netbox_importer/generator/nautobot.py
def get_or_create_wrapper(self, content_type: ContentTypeStr) -> "NautobotModelWrapper":
    """Get or create a Nautobot model wrapper."""
    if content_type in self.wrappers:
        return self.wrappers[content_type]

    return NautobotModelWrapper(self, content_type)

PreImportResult

Bases: Enum

Pre Import Response.

Source code in nautobot_netbox_importer/generator/source.py
class PreImportResult(Enum):
    """Pre Import Response."""

    SKIP_RECORD = False
    USE_RECORD = True

SourceAdapter

Bases: BaseAdapter

Source DiffSync Adapter.

Source code in nautobot_netbox_importer/generator/source.py
class SourceAdapter(BaseAdapter):
    """Source DiffSync Adapter."""

    def __init__(
        self,
        *args,
        get_source_data: SourceDataGenerator,
        trace_issues: bool = False,
        nautobot: Optional[NautobotAdapter] = None,
        logger=None,
        **kwargs,
    ):
        """Initialize the SourceAdapter."""
        super().__init__(*args, **kwargs)

        self.get_source_data = get_source_data
        self.wrappers: OrderedDict[ContentTypeStr, SourceModelWrapper] = OrderedDict()
        self.nautobot = nautobot or NautobotAdapter()
        self.nautobot.trace_issues = trace_issues
        self.content_type_ids_mapping: Dict[int, SourceModelWrapper] = {}
        self.logger = logger or default_logger
        self.summary = ImportSummary()

        # From Nautobot to Source content type mapping
        # When multiple source content types are mapped to the single nautobot content type, mapping is set to `None`
        self._content_types_back_mapping: Dict[ContentTypeStr, Optional[ContentTypeStr]] = {}

    # pylint: disable=too-many-arguments,too-many-branches,too-many-locals
    def configure_model(
        self,
        content_type: ContentTypeStr,
        nautobot_content_type: ContentTypeStr = "",
        extend_content_type: ContentTypeStr = "",
        identifiers: Optional[Iterable[FieldName]] = None,
        fields: Optional[Mapping[FieldName, SourceFieldDefinition]] = None,
        default_reference: Optional[RecordData] = None,
        flags: Optional[DiffSyncModelFlags] = None,
        nautobot_flags: Optional[DiffSyncModelFlags] = None,
        pre_import: Optional[PreImport] = None,
        disable_related_reference: Optional[bool] = None,
        forward_references: Optional[ForwardReferences] = None,
    ) -> "SourceModelWrapper":
        """Create if not exist and configure a wrapper for a given source content type.

        Create Nautobot content type wrapper as well.
        """
        content_type = content_type.lower()
        nautobot_content_type = nautobot_content_type.lower()
        extend_content_type = extend_content_type.lower()

        if extend_content_type:
            if nautobot_content_type:
                raise ValueError(f"Can't specify both nautobot_content_type and extend_content_type {content_type}")
            extends_wrapper = self.wrappers[extend_content_type]
            nautobot_content_type = extends_wrapper.nautobot.content_type
        else:
            extends_wrapper = None

        if content_type in self.wrappers:
            wrapper = self.wrappers[content_type]
            if nautobot_content_type and wrapper.nautobot.content_type != nautobot_content_type:
                raise ValueError(
                    f"Content type {content_type} already mapped to {wrapper.nautobot.content_type} "
                    f"can't map to {nautobot_content_type}"
                )
        else:
            nautobot_wrapper = self.nautobot.get_or_create_wrapper(nautobot_content_type or content_type)
            wrapper = SourceModelWrapper(self, content_type, nautobot_wrapper)
            if not extends_wrapper:
                if nautobot_wrapper.content_type in self._content_types_back_mapping:
                    if self._content_types_back_mapping[nautobot_wrapper.content_type] != content_type:
                        self._content_types_back_mapping[nautobot_wrapper.content_type] = None
                else:
                    self._content_types_back_mapping[nautobot_wrapper.content_type] = content_type

        if extends_wrapper:
            wrapper.extends_wrapper = extends_wrapper

        if identifiers:
            wrapper.set_identifiers(identifiers)
        for field_name, definition in (fields or {}).items():
            wrapper.add_field(field_name, SourceFieldSource.CUSTOM).set_definition(definition)
        if default_reference:
            wrapper.set_default_reference(default_reference)
        if flags is not None:
            wrapper.flags = flags
        if nautobot_flags is not None:
            wrapper.nautobot.flags = nautobot_flags
        if pre_import:
            wrapper.pre_import = pre_import
        if disable_related_reference is not None:
            wrapper.disable_related_reference = disable_related_reference
        if forward_references:
            wrapper.forward_references = forward_references

        return wrapper

    def disable_model(self, content_type: ContentTypeStr, disable_reason: str) -> None:
        """Disable model importing."""
        self.get_or_create_wrapper(content_type).disable_reason = disable_reason

    def summarize(self, diffsync_summary: DiffSyncSummary) -> None:
        """Summarize the import."""
        self.summary.diffsync = diffsync_summary

        wrapper_to_id = {value: key for key, value in self.content_type_ids_mapping.items()}

        for content_type in sorted(self.wrappers):
            wrapper = self.wrappers.get(content_type)
            if wrapper:
                self.summary.source.append(wrapper.get_summary(wrapper_to_id.get(wrapper, None)))

        for content_type in sorted(self.nautobot.wrappers):
            wrapper = self.nautobot.wrappers.get(content_type)
            if wrapper:
                self.summary.nautobot.append(wrapper.get_summary())

    def get_or_create_wrapper(self, value: Union[None, SourceContentType]) -> "SourceModelWrapper":
        """Get a source Wrapper for a given content type."""
        # Enable mapping back from Nautobot content type, when using Nautobot model or wrapper
        map_back = False

        if not value:
            raise ValueError("Missing value")

        if isinstance(value, SourceModelWrapper):
            return value

        if isinstance(value, type(NautobotBaseModel)):
            map_back = True
            value = value._meta.label.lower()  # type: ignore
        elif isinstance(value, NautobotModelWrapper):
            map_back = True
            value = value.content_type

        if isinstance(value, str):
            value = value.lower()
        elif isinstance(value, int):
            if value not in self.content_type_ids_mapping:
                raise ValueError(f"Content type not found {value}")
            return self.content_type_ids_mapping[value]
        elif isinstance(value, Iterable) and len(value) == 2:  # noqa: PLR2004
            value = ".".join(value).lower()
        else:
            raise ValueError(f"Invalid content type {value}")

        if map_back and value in self._content_types_back_mapping:
            back_mapping = self._content_types_back_mapping.get(value, None)
            if not back_mapping:
                raise ValueError(f"Ambiguous content type back mapping {value}")
            value = back_mapping

        if value in self.wrappers:
            return self.wrappers[value]

        return self.configure_model(value)

    def get_nautobot_content_type_uid(self, content_type: ContentTypeValue) -> int:
        """Get the Django content type ID for a given content type."""
        if isinstance(content_type, int):
            wrapper = self.content_type_ids_mapping.get(content_type, None)
            if not wrapper:
                raise ValueError(f"Content type not found {content_type}")
            return wrapper.nautobot.content_type_instance.pk
        if not isinstance(content_type, str):
            if not len(content_type) == 2:
                raise ValueError(f"Invalid content type {content_type}")
            content_type = ".".join(content_type)

        wrapper = self.get_or_create_wrapper(content_type)

        return wrapper.nautobot.content_type_instance.pk

    def load(self) -> None:
        """Load data from the source."""
        self.import_data()
        self.post_import()

    def import_data(self) -> None:
        """Import data from the source."""
        get_source_data = self.get_source_data

        # First pass to enhance pre-defined wrappers structure
        for content_type, data in get_source_data():
            if content_type in self.wrappers:
                wrapper = self.wrappers[content_type]
            else:
                wrapper = self.configure_model(content_type)
            wrapper.first_pass(data)

        # Create importers, wrappers structure is updated as needed
        while True:
            wrappers = [
                wrapper
                for wrapper in self.wrappers.values()
                if wrapper.importers is None and not wrapper.disable_reason
            ]
            if not wrappers:
                break
            for wrapper in wrappers:
                wrapper.create_importers()

        # Second pass to import actual data
        for content_type, data in get_source_data():
            self.wrappers[content_type].second_pass(data)

    def post_import(self) -> None:
        """Post import processing."""
        while any(wrapper.post_import() for wrapper in self.wrappers.values()):
            pass

        for nautobot_wrapper in self.get_imported_nautobot_wrappers():
            diffsync_class = nautobot_wrapper.diffsync_class
            # pylint: disable=protected-access
            model_name = diffsync_class._modelname
            self.top_level.append(model_name)
            setattr(self, model_name, diffsync_class)
            setattr(self.nautobot, model_name, getattr(self, model_name))

    def get_imported_nautobot_wrappers(self) -> Generator[NautobotModelWrapper, None, None]:
        """Get a list of Nautobot model wrappers in the order of import."""
        result = OrderedDict()

        for wrapper in self.wrappers.values():
            if (
                wrapper
                and not wrapper.disable_reason
                and wrapper.stats.created > 0
                and wrapper.nautobot.content_type not in result
            ):
                result[wrapper.nautobot.content_type] = wrapper.nautobot

        for content_type in IMPORT_ORDER:
            if content_type in result:
                yield result[content_type]
                del result[content_type]

        yield from result.values()
__init__(*args, get_source_data, trace_issues=False, nautobot=None, logger=None, **kwargs)

Initialize the SourceAdapter.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(
    self,
    *args,
    get_source_data: SourceDataGenerator,
    trace_issues: bool = False,
    nautobot: Optional[NautobotAdapter] = None,
    logger=None,
    **kwargs,
):
    """Initialize the SourceAdapter."""
    super().__init__(*args, **kwargs)

    self.get_source_data = get_source_data
    self.wrappers: OrderedDict[ContentTypeStr, SourceModelWrapper] = OrderedDict()
    self.nautobot = nautobot or NautobotAdapter()
    self.nautobot.trace_issues = trace_issues
    self.content_type_ids_mapping: Dict[int, SourceModelWrapper] = {}
    self.logger = logger or default_logger
    self.summary = ImportSummary()

    # From Nautobot to Source content type mapping
    # When multiple source content types are mapped to the single nautobot content type, mapping is set to `None`
    self._content_types_back_mapping: Dict[ContentTypeStr, Optional[ContentTypeStr]] = {}
configure_model(content_type, nautobot_content_type='', extend_content_type='', identifiers=None, fields=None, default_reference=None, flags=None, nautobot_flags=None, pre_import=None, disable_related_reference=None, forward_references=None)

Create if not exist and configure a wrapper for a given source content type.

Create Nautobot content type wrapper as well.

Source code in nautobot_netbox_importer/generator/source.py
def configure_model(
    self,
    content_type: ContentTypeStr,
    nautobot_content_type: ContentTypeStr = "",
    extend_content_type: ContentTypeStr = "",
    identifiers: Optional[Iterable[FieldName]] = None,
    fields: Optional[Mapping[FieldName, SourceFieldDefinition]] = None,
    default_reference: Optional[RecordData] = None,
    flags: Optional[DiffSyncModelFlags] = None,
    nautobot_flags: Optional[DiffSyncModelFlags] = None,
    pre_import: Optional[PreImport] = None,
    disable_related_reference: Optional[bool] = None,
    forward_references: Optional[ForwardReferences] = None,
) -> "SourceModelWrapper":
    """Create if not exist and configure a wrapper for a given source content type.

    Create Nautobot content type wrapper as well.
    """
    content_type = content_type.lower()
    nautobot_content_type = nautobot_content_type.lower()
    extend_content_type = extend_content_type.lower()

    if extend_content_type:
        if nautobot_content_type:
            raise ValueError(f"Can't specify both nautobot_content_type and extend_content_type {content_type}")
        extends_wrapper = self.wrappers[extend_content_type]
        nautobot_content_type = extends_wrapper.nautobot.content_type
    else:
        extends_wrapper = None

    if content_type in self.wrappers:
        wrapper = self.wrappers[content_type]
        if nautobot_content_type and wrapper.nautobot.content_type != nautobot_content_type:
            raise ValueError(
                f"Content type {content_type} already mapped to {wrapper.nautobot.content_type} "
                f"can't map to {nautobot_content_type}"
            )
    else:
        nautobot_wrapper = self.nautobot.get_or_create_wrapper(nautobot_content_type or content_type)
        wrapper = SourceModelWrapper(self, content_type, nautobot_wrapper)
        if not extends_wrapper:
            if nautobot_wrapper.content_type in self._content_types_back_mapping:
                if self._content_types_back_mapping[nautobot_wrapper.content_type] != content_type:
                    self._content_types_back_mapping[nautobot_wrapper.content_type] = None
            else:
                self._content_types_back_mapping[nautobot_wrapper.content_type] = content_type

    if extends_wrapper:
        wrapper.extends_wrapper = extends_wrapper

    if identifiers:
        wrapper.set_identifiers(identifiers)
    for field_name, definition in (fields or {}).items():
        wrapper.add_field(field_name, SourceFieldSource.CUSTOM).set_definition(definition)
    if default_reference:
        wrapper.set_default_reference(default_reference)
    if flags is not None:
        wrapper.flags = flags
    if nautobot_flags is not None:
        wrapper.nautobot.flags = nautobot_flags
    if pre_import:
        wrapper.pre_import = pre_import
    if disable_related_reference is not None:
        wrapper.disable_related_reference = disable_related_reference
    if forward_references:
        wrapper.forward_references = forward_references

    return wrapper
disable_model(content_type, disable_reason)

Disable model importing.

Source code in nautobot_netbox_importer/generator/source.py
def disable_model(self, content_type: ContentTypeStr, disable_reason: str) -> None:
    """Disable model importing."""
    self.get_or_create_wrapper(content_type).disable_reason = disable_reason
get_imported_nautobot_wrappers()

Get a list of Nautobot model wrappers in the order of import.

Source code in nautobot_netbox_importer/generator/source.py
def get_imported_nautobot_wrappers(self) -> Generator[NautobotModelWrapper, None, None]:
    """Get a list of Nautobot model wrappers in the order of import."""
    result = OrderedDict()

    for wrapper in self.wrappers.values():
        if (
            wrapper
            and not wrapper.disable_reason
            and wrapper.stats.created > 0
            and wrapper.nautobot.content_type not in result
        ):
            result[wrapper.nautobot.content_type] = wrapper.nautobot

    for content_type in IMPORT_ORDER:
        if content_type in result:
            yield result[content_type]
            del result[content_type]

    yield from result.values()
get_nautobot_content_type_uid(content_type)

Get the Django content type ID for a given content type.

Source code in nautobot_netbox_importer/generator/source.py
def get_nautobot_content_type_uid(self, content_type: ContentTypeValue) -> int:
    """Get the Django content type ID for a given content type."""
    if isinstance(content_type, int):
        wrapper = self.content_type_ids_mapping.get(content_type, None)
        if not wrapper:
            raise ValueError(f"Content type not found {content_type}")
        return wrapper.nautobot.content_type_instance.pk
    if not isinstance(content_type, str):
        if not len(content_type) == 2:
            raise ValueError(f"Invalid content type {content_type}")
        content_type = ".".join(content_type)

    wrapper = self.get_or_create_wrapper(content_type)

    return wrapper.nautobot.content_type_instance.pk
get_or_create_wrapper(value)

Get a source Wrapper for a given content type.

Source code in nautobot_netbox_importer/generator/source.py
def get_or_create_wrapper(self, value: Union[None, SourceContentType]) -> "SourceModelWrapper":
    """Get a source Wrapper for a given content type."""
    # Enable mapping back from Nautobot content type, when using Nautobot model or wrapper
    map_back = False

    if not value:
        raise ValueError("Missing value")

    if isinstance(value, SourceModelWrapper):
        return value

    if isinstance(value, type(NautobotBaseModel)):
        map_back = True
        value = value._meta.label.lower()  # type: ignore
    elif isinstance(value, NautobotModelWrapper):
        map_back = True
        value = value.content_type

    if isinstance(value, str):
        value = value.lower()
    elif isinstance(value, int):
        if value not in self.content_type_ids_mapping:
            raise ValueError(f"Content type not found {value}")
        return self.content_type_ids_mapping[value]
    elif isinstance(value, Iterable) and len(value) == 2:  # noqa: PLR2004
        value = ".".join(value).lower()
    else:
        raise ValueError(f"Invalid content type {value}")

    if map_back and value in self._content_types_back_mapping:
        back_mapping = self._content_types_back_mapping.get(value, None)
        if not back_mapping:
            raise ValueError(f"Ambiguous content type back mapping {value}")
        value = back_mapping

    if value in self.wrappers:
        return self.wrappers[value]

    return self.configure_model(value)
import_data()

Import data from the source.

Source code in nautobot_netbox_importer/generator/source.py
def import_data(self) -> None:
    """Import data from the source."""
    get_source_data = self.get_source_data

    # First pass to enhance pre-defined wrappers structure
    for content_type, data in get_source_data():
        if content_type in self.wrappers:
            wrapper = self.wrappers[content_type]
        else:
            wrapper = self.configure_model(content_type)
        wrapper.first_pass(data)

    # Create importers, wrappers structure is updated as needed
    while True:
        wrappers = [
            wrapper
            for wrapper in self.wrappers.values()
            if wrapper.importers is None and not wrapper.disable_reason
        ]
        if not wrappers:
            break
        for wrapper in wrappers:
            wrapper.create_importers()

    # Second pass to import actual data
    for content_type, data in get_source_data():
        self.wrappers[content_type].second_pass(data)
load()

Load data from the source.

Source code in nautobot_netbox_importer/generator/source.py
def load(self) -> None:
    """Load data from the source."""
    self.import_data()
    self.post_import()
post_import()

Post import processing.

Source code in nautobot_netbox_importer/generator/source.py
def post_import(self) -> None:
    """Post import processing."""
    while any(wrapper.post_import() for wrapper in self.wrappers.values()):
        pass

    for nautobot_wrapper in self.get_imported_nautobot_wrappers():
        diffsync_class = nautobot_wrapper.diffsync_class
        # pylint: disable=protected-access
        model_name = diffsync_class._modelname
        self.top_level.append(model_name)
        setattr(self, model_name, diffsync_class)
        setattr(self.nautobot, model_name, getattr(self, model_name))
summarize(diffsync_summary)

Summarize the import.

Source code in nautobot_netbox_importer/generator/source.py
def summarize(self, diffsync_summary: DiffSyncSummary) -> None:
    """Summarize the import."""
    self.summary.diffsync = diffsync_summary

    wrapper_to_id = {value: key for key, value in self.content_type_ids_mapping.items()}

    for content_type in sorted(self.wrappers):
        wrapper = self.wrappers.get(content_type)
        if wrapper:
            self.summary.source.append(wrapper.get_summary(wrapper_to_id.get(wrapper, None)))

    for content_type in sorted(self.nautobot.wrappers):
        wrapper = self.nautobot.wrappers.get(content_type)
        if wrapper:
            self.summary.nautobot.append(wrapper.get_summary())

SourceField

Source Field.

Source code in nautobot_netbox_importer/generator/source.py
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
class SourceField:
    """Source Field."""

    def __init__(self, wrapper: SourceModelWrapper, name: FieldName, source: SourceFieldSource):
        """Initialize the SourceField."""
        self.wrapper = wrapper
        wrapper.fields[name] = self
        self.name = name
        self.definition: SourceFieldDefinition = name
        self.sources = set((source,))
        self.processed = False
        self._nautobot: Optional[NautobotField] = None
        self.importer: Optional[SourceFieldImporter] = None
        self.default_value: Any = None
        self.disable_reason: str = ""

    def __str__(self) -> str:
        """Return a string representation of the field."""
        return self.wrapper.format_field_name(self.name)

    @property
    def nautobot(self) -> NautobotField:
        """Get the Nautobot field wrapper."""
        if not self._nautobot:
            raise RuntimeError(f"Missing Nautobot field for {self}")
        return self._nautobot

    def get_summary(self) -> FieldSummary:
        """Get a summary of the field."""
        return FieldSummary(
            name=self.name,
            nautobot_name=self._nautobot and self._nautobot.name,
            nautobot_internal_type=self._nautobot and self._nautobot.internal_type.value,
            nautobot_can_import=self._nautobot and self._nautobot.can_import,
            importer=self.importer and self.importer.__name__,
            definition=serialize_to_summary(self.definition),
            sources=sorted(source.name for source in self.sources),
            default_value=serialize_to_summary(self.default_value),
            disable_reason=self.disable_reason,
            required=self._nautobot.required if self._nautobot else False,
        )

    def disable(self, reason: str) -> None:
        """Disable field importing."""
        self.definition = None
        self.importer = None
        self.processed = True
        self.disable_reason = reason

    def handle_sibling(self, sibling: Union["SourceField", FieldName], nautobot_name: FieldName = "") -> "SourceField":
        """Specify, that this field importer handles other field."""
        if not self.importer:
            raise RuntimeError(f"Call `handle sibling` after setting importer for {self}")

        if isinstance(sibling, FieldName):
            sibling = self.wrapper.add_field(sibling, SourceFieldSource.SIBLING)

        sibling.set_nautobot_field(nautobot_name or self.nautobot.name)
        sibling.importer = self.importer
        sibling.processed = True

        if self.nautobot.can_import and not sibling.nautobot.can_import:
            self.disable(f"Can't import {self} based on {sibling}")

        return sibling

    def add_issue(self, issue_type: str, message: str, target: Optional[DiffSyncModel] = None) -> None:
        """Add an importer issue to the Nautobot Model Wrapper."""
        self.wrapper.nautobot.add_issue(issue_type, message=str({self.name: message}), diffsync_instance=target)

    def set_definition(self, definition: SourceFieldDefinition) -> None:
        """Customize field definition."""
        if self.processed:
            raise RuntimeError(f"Field already processed. {self}")

        if self.definition != definition:
            if self.definition != self.name:
                self.add_issue(
                    "OverrideDefinition",
                    f"Overriding field definition | Original: `{self.definition}` | New: `{definition}`",
                )
            self.definition = definition

    def create_importer(self) -> None:
        """Create importer for the field."""
        if self.processed:
            return
        self.processed = True

        if self.definition is None:
            return

        if isinstance(self.definition, FieldName):
            self.set_importer(nautobot_name=self.definition)
        elif callable(self.definition):
            self.definition(self)
        else:
            raise NotImplementedError(f"Unsupported field definition {self.definition}")

    def get_source_value(self, source: RecordData) -> Any:
        """Get a value from the source data, returning a default value if the value is empty."""
        if self.name not in source:
            return self.default_value

        result = source[self.name]
        return self.default_value if result in EMPTY_VALUES else result

    def set_nautobot_value(self, target: DiffSyncModel, value: Any) -> None:
        """Set a value to the Nautobot model."""
        if value in EMPTY_VALUES:
            if hasattr(target, self.nautobot.name):
                delattr(target, self.nautobot.name)
        else:
            setattr(target, self.nautobot.name, value)

    def set_nautobot_field(self, nautobot_name: FieldName = "") -> NautobotField:
        """Set a Nautobot field name for the field."""
        result = self.wrapper.nautobot.add_field(nautobot_name or self.name)
        if result.field:
            default_value = getattr(result.field, "default", None)
            if default_value not in EMPTY_VALUES and not isinstance(default_value, Callable):
                self.default_value = default_value
        self._nautobot = result
        if result.name == "last_updated":
            self.disable("Last updated field is updated with each write")
        return result

    # pylint: disable=too-many-branches
    def set_importer(
        self,
        importer: Optional[SourceFieldImporter] = None,
        nautobot_name: Optional[FieldName] = "",
        override=False,
    ) -> Optional[SourceFieldImporter]:
        """Sets the importer and Nautobot field if not already specified.

        If `nautobot_name` is not provided, the field name is used.

        Passing None to `nautobot_name` indicates that there is custom mapping without a direct relationship to a Nautobot field.
        """
        if self.disable_reason:
            raise RuntimeError(f"Can't set importer for disabled {self}")
        if self.importer and not override:
            raise RuntimeError(f"Importer already set for {self}")
        if not self._nautobot and nautobot_name is not None:
            self.set_nautobot_field(nautobot_name)

        if importer:
            self.importer = importer
            return importer

        if self.disable_reason or not self.nautobot.can_import:
            return None

        internal_type = self.nautobot.internal_type

        if internal_type == InternalFieldType.JSON_FIELD:
            self.set_json_importer()
        elif internal_type == InternalFieldType.DATE_FIELD:
            self.set_date_importer()
        elif internal_type == InternalFieldType.DATE_TIME_FIELD:
            self.set_datetime_importer()
        elif internal_type == InternalFieldType.UUID_FIELD:
            self.set_uuid_importer()
        elif internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
            self.set_m2m_importer()
        elif internal_type == InternalFieldType.STATUS_FIELD:
            self.set_status_importer()
        elif self.nautobot.is_reference:
            self.set_relation_importer()
        elif getattr(self.nautobot.field, "choices", None):
            self.set_choice_importer()
        elif self.nautobot.is_integer:
            self.set_integer_importer()
        else:
            self.set_value_importer()

        return self.importer

    def set_value_importer(self) -> None:
        """Set a value importer."""

        def value_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = self.get_source_value(source)
            self.set_nautobot_value(target, value)

        self.set_importer(value_importer)

    def set_json_importer(self) -> None:
        """Set a JSON field importer."""

        def json_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(self.name, None)
            if isinstance(value, str) and value:
                value = json.loads(value)
            self.set_nautobot_value(target, value)

        self.set_importer(json_importer)

    def set_choice_importer(self) -> None:
        """Set a choice field importer."""
        field_choices = getattr(self.nautobot.field, "choices", None)
        if not field_choices:
            raise ValueError(f"Invalid field_choices for {self}")

        choices = dict(get_field_choices(field_choices))

        def choice_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = self.get_source_value(source)
            if value in choices:
                self.set_nautobot_value(target, value)
            elif self.nautobot.required:
                # Set the choice value even it's not valid in Nautobot as it's required
                self.set_nautobot_value(target, value)
                raise InvalidChoiceValueIssue(self, value)
            elif value in EMPTY_VALUES:
                self.set_nautobot_value(target, value)
            else:
                raise InvalidChoiceValueIssue(self, value, None)

        self.set_importer(choice_importer)

    def set_integer_importer(self) -> None:
        """Set an integer field importer."""

        def integer_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            source_value = self.get_source_value(source)
            if source_value in EMPTY_VALUES:
                self.set_nautobot_value(target, source_value)
            else:
                source_value = float(source_value)
                value = int(source_value)
                self.set_nautobot_value(target, value)
                if value != source_value:
                    raise SourceFieldImporterIssue(f"Invalid source value {source_value}, truncated to {value}", self)

        self.set_importer(integer_importer)

    def _get_related_wrapper(self, related_source: Optional[SourceContentType]) -> SourceModelWrapper:
        """Get a related wrapper."""
        if related_source:
            return self.wrapper.adapter.get_or_create_wrapper(related_source)

        if self.name == "parent":
            return self.wrapper

        return self.wrapper.adapter.get_or_create_wrapper(self.nautobot.related_model)

    def set_relation_importer(self, related_source: Optional[SourceContentType] = None) -> None:
        """Set a relation importer."""
        related_wrapper = self._get_related_wrapper(related_source)

        if self.nautobot.is_content_type:
            self.set_content_type_importer()
            return

        if self.default_value in EMPTY_VALUES and related_wrapper.default_reference_uid:
            self.default_value = related_wrapper.default_reference_uid

        if not (self.default_value is None or isinstance(self.default_value, UUID)):
            raise NotImplementedError(f"Default value {self.default_value} is not a UUID")

        def relation_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = self.get_source_value(source)
            if value in EMPTY_VALUES:
                self.set_nautobot_value(target, value)
            else:
                if isinstance(value, (UUID, str, int)):
                    result = related_wrapper.get_pk_from_uid(value)
                else:
                    result = related_wrapper.get_pk_from_identifiers(value)
                self.set_nautobot_value(target, result)
                self.wrapper.add_reference(related_wrapper, result)

        self.set_importer(relation_importer)

    def set_content_type_importer(self) -> None:
        """Set a content type importer."""
        adapter = self.wrapper.adapter

        def content_type_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            content_type = source.get(self.name, None)
            if content_type not in EMPTY_VALUES:
                content_type = adapter.get_nautobot_content_type_uid(content_type)
            self.set_nautobot_value(target, content_type)

        self.set_importer(content_type_importer)

    def set_m2m_importer(self, related_source: Optional[SourceContentType] = None) -> None:
        """Set a many to many importer."""
        if not isinstance(self.nautobot.field, DjangoField):
            raise NotImplementedError(f"Unsupported m2m importer {self}")

        related_wrapper = self._get_related_wrapper(related_source)

        if related_wrapper.content_type == "contenttypes.contenttype":
            self.set_content_types_importer()
            return

        def m2m_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            values = source.get(self.name, None)
            if values in EMPTY_VALUES:
                return

            if isinstance(values, (UUID, str, int)):
                result = related_wrapper.get_pk_from_uid(values)
                self.wrapper.add_reference(related_wrapper, result)
                self.set_nautobot_value(target, {result})
                return

            if not isinstance(values, (list, set)):
                raise ValueError(f"Invalid value {values} for field {self.name}")

            results = set()
            for value in values:
                if isinstance(value, (UUID, str, int)):
                    result = related_wrapper.get_pk_from_uid(value)
                else:
                    result = related_wrapper.get_pk_from_identifiers(value)

                results.add(result)
                self.wrapper.add_reference(related_wrapper, result)

            self.set_nautobot_value(target, results)

        self.set_importer(m2m_importer)

    def set_content_types_importer(self) -> None:
        """Set a content types importer."""
        adapter = self.wrapper.adapter

        def content_types_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            values = source.get(self.name, None)
            if values in EMPTY_VALUES:
                return

            if not isinstance(values, (list, set)):
                raise ValueError(f"Invalid value {values} for field {self.name}")

            nautobot_values = set()
            for item in values:
                try:
                    nautobot_values.add(adapter.get_nautobot_content_type_uid(item))
                except NautobotModelNotFound:
                    self.add_issue("InvalidContentType", f"Invalid content type {item}, skipping", target)

            self.set_nautobot_value(target, nautobot_values)

        self.set_importer(content_types_importer)

    def set_datetime_importer(self) -> None:
        """Set a datetime importer."""

        def datetime_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(self.name, None)
            if value not in EMPTY_VALUES:
                value = normalize_datetime(value)
            self.set_nautobot_value(target, value)

        self.set_importer(datetime_importer)

    def set_relation_and_type_importer(self, type_field: "SourceField") -> None:
        """Set a relation UUID importer based on the type field."""

        def relation_and_type_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            source_uid = source.get(self.name, None)
            source_type = source.get(type_field.name, None)
            if source_type in EMPTY_VALUES or source_uid in EMPTY_VALUES:
                if source_uid not in EMPTY_VALUES or source_type not in EMPTY_VALUES:
                    raise ValueError(
                        f"Both {self}=`{source_uid}` and {type_field}=`{source_type}` must be empty or not empty."
                    )
                return

            type_wrapper = self.wrapper.adapter.get_or_create_wrapper(source_type)
            uid = type_wrapper.get_pk_from_uid(source_uid)
            self.set_nautobot_value(target, uid)
            type_field.set_nautobot_value(target, type_wrapper.nautobot.content_type_instance.pk)
            self.wrapper.add_reference(type_wrapper, uid)

        self.set_importer(relation_and_type_importer)
        self.handle_sibling(type_field, type_field.name)

    def set_uuid_importer(self) -> None:
        """Set an UUID importer."""
        if self.name.endswith("_id"):
            type_field = self.wrapper.fields.get(self.name[:-3] + "_type", None)
            if type_field and type_field.nautobot.is_content_type:
                # Handles `<field name>_id` and `<field name>_type` fields combination
                self.set_relation_and_type_importer(type_field)
                return

        def uuid_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(self.name, None)
            if value not in EMPTY_VALUES:
                value = UUID(value)
            self.set_nautobot_value(target, value)

        self.set_importer(uuid_importer)

    def set_date_importer(self) -> None:
        """Set a date importer."""

        def date_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(self.name, None)
            if value not in EMPTY_VALUES and not isinstance(value, datetime.date):
                value = datetime.date.fromisoformat(str(value))
            self.set_nautobot_value(target, value)

        self.set_importer(date_importer)

    def set_status_importer(self) -> None:
        """Set a status importer."""
        status_wrapper = self.wrapper.adapter.get_or_create_wrapper("extras.status")
        if not self.default_value:
            self.default_value = status_wrapper.default_reference_uid

        if not (self.default_value is None or isinstance(self.default_value, UUID)):
            raise NotImplementedError(f"Default value {self.default_value} is not a UUID")

        def status_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            status = source.get(self.name, None)
            if status:
                value = status_wrapper.cache_record({"name": status[0].upper() + status[1:]})
            else:
                value = self.default_value

            self.set_nautobot_value(target, value)
            if value:
                self.wrapper.add_reference(status_wrapper, value)

        self.set_importer(status_importer)
nautobot: NautobotField property

Get the Nautobot field wrapper.

__init__(wrapper, name, source)

Initialize the SourceField.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(self, wrapper: SourceModelWrapper, name: FieldName, source: SourceFieldSource):
    """Initialize the SourceField."""
    self.wrapper = wrapper
    wrapper.fields[name] = self
    self.name = name
    self.definition: SourceFieldDefinition = name
    self.sources = set((source,))
    self.processed = False
    self._nautobot: Optional[NautobotField] = None
    self.importer: Optional[SourceFieldImporter] = None
    self.default_value: Any = None
    self.disable_reason: str = ""
__str__()

Return a string representation of the field.

Source code in nautobot_netbox_importer/generator/source.py
def __str__(self) -> str:
    """Return a string representation of the field."""
    return self.wrapper.format_field_name(self.name)
add_issue(issue_type, message, target=None)

Add an importer issue to the Nautobot Model Wrapper.

Source code in nautobot_netbox_importer/generator/source.py
def add_issue(self, issue_type: str, message: str, target: Optional[DiffSyncModel] = None) -> None:
    """Add an importer issue to the Nautobot Model Wrapper."""
    self.wrapper.nautobot.add_issue(issue_type, message=str({self.name: message}), diffsync_instance=target)
create_importer()

Create importer for the field.

Source code in nautobot_netbox_importer/generator/source.py
def create_importer(self) -> None:
    """Create importer for the field."""
    if self.processed:
        return
    self.processed = True

    if self.definition is None:
        return

    if isinstance(self.definition, FieldName):
        self.set_importer(nautobot_name=self.definition)
    elif callable(self.definition):
        self.definition(self)
    else:
        raise NotImplementedError(f"Unsupported field definition {self.definition}")
disable(reason)

Disable field importing.

Source code in nautobot_netbox_importer/generator/source.py
def disable(self, reason: str) -> None:
    """Disable field importing."""
    self.definition = None
    self.importer = None
    self.processed = True
    self.disable_reason = reason
get_source_value(source)

Get a value from the source data, returning a default value if the value is empty.

Source code in nautobot_netbox_importer/generator/source.py
def get_source_value(self, source: RecordData) -> Any:
    """Get a value from the source data, returning a default value if the value is empty."""
    if self.name not in source:
        return self.default_value

    result = source[self.name]
    return self.default_value if result in EMPTY_VALUES else result
get_summary()

Get a summary of the field.

Source code in nautobot_netbox_importer/generator/source.py
def get_summary(self) -> FieldSummary:
    """Get a summary of the field."""
    return FieldSummary(
        name=self.name,
        nautobot_name=self._nautobot and self._nautobot.name,
        nautobot_internal_type=self._nautobot and self._nautobot.internal_type.value,
        nautobot_can_import=self._nautobot and self._nautobot.can_import,
        importer=self.importer and self.importer.__name__,
        definition=serialize_to_summary(self.definition),
        sources=sorted(source.name for source in self.sources),
        default_value=serialize_to_summary(self.default_value),
        disable_reason=self.disable_reason,
        required=self._nautobot.required if self._nautobot else False,
    )
handle_sibling(sibling, nautobot_name='')

Specify, that this field importer handles other field.

Source code in nautobot_netbox_importer/generator/source.py
def handle_sibling(self, sibling: Union["SourceField", FieldName], nautobot_name: FieldName = "") -> "SourceField":
    """Specify, that this field importer handles other field."""
    if not self.importer:
        raise RuntimeError(f"Call `handle sibling` after setting importer for {self}")

    if isinstance(sibling, FieldName):
        sibling = self.wrapper.add_field(sibling, SourceFieldSource.SIBLING)

    sibling.set_nautobot_field(nautobot_name or self.nautobot.name)
    sibling.importer = self.importer
    sibling.processed = True

    if self.nautobot.can_import and not sibling.nautobot.can_import:
        self.disable(f"Can't import {self} based on {sibling}")

    return sibling
set_choice_importer()

Set a choice field importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_choice_importer(self) -> None:
    """Set a choice field importer."""
    field_choices = getattr(self.nautobot.field, "choices", None)
    if not field_choices:
        raise ValueError(f"Invalid field_choices for {self}")

    choices = dict(get_field_choices(field_choices))

    def choice_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = self.get_source_value(source)
        if value in choices:
            self.set_nautobot_value(target, value)
        elif self.nautobot.required:
            # Set the choice value even it's not valid in Nautobot as it's required
            self.set_nautobot_value(target, value)
            raise InvalidChoiceValueIssue(self, value)
        elif value in EMPTY_VALUES:
            self.set_nautobot_value(target, value)
        else:
            raise InvalidChoiceValueIssue(self, value, None)

    self.set_importer(choice_importer)
set_content_type_importer()

Set a content type importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_content_type_importer(self) -> None:
    """Set a content type importer."""
    adapter = self.wrapper.adapter

    def content_type_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        content_type = source.get(self.name, None)
        if content_type not in EMPTY_VALUES:
            content_type = adapter.get_nautobot_content_type_uid(content_type)
        self.set_nautobot_value(target, content_type)

    self.set_importer(content_type_importer)
set_content_types_importer()

Set a content types importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_content_types_importer(self) -> None:
    """Set a content types importer."""
    adapter = self.wrapper.adapter

    def content_types_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        values = source.get(self.name, None)
        if values in EMPTY_VALUES:
            return

        if not isinstance(values, (list, set)):
            raise ValueError(f"Invalid value {values} for field {self.name}")

        nautobot_values = set()
        for item in values:
            try:
                nautobot_values.add(adapter.get_nautobot_content_type_uid(item))
            except NautobotModelNotFound:
                self.add_issue("InvalidContentType", f"Invalid content type {item}, skipping", target)

        self.set_nautobot_value(target, nautobot_values)

    self.set_importer(content_types_importer)
set_date_importer()

Set a date importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_date_importer(self) -> None:
    """Set a date importer."""

    def date_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = source.get(self.name, None)
        if value not in EMPTY_VALUES and not isinstance(value, datetime.date):
            value = datetime.date.fromisoformat(str(value))
        self.set_nautobot_value(target, value)

    self.set_importer(date_importer)
set_datetime_importer()

Set a datetime importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_datetime_importer(self) -> None:
    """Set a datetime importer."""

    def datetime_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = source.get(self.name, None)
        if value not in EMPTY_VALUES:
            value = normalize_datetime(value)
        self.set_nautobot_value(target, value)

    self.set_importer(datetime_importer)
set_definition(definition)

Customize field definition.

Source code in nautobot_netbox_importer/generator/source.py
def set_definition(self, definition: SourceFieldDefinition) -> None:
    """Customize field definition."""
    if self.processed:
        raise RuntimeError(f"Field already processed. {self}")

    if self.definition != definition:
        if self.definition != self.name:
            self.add_issue(
                "OverrideDefinition",
                f"Overriding field definition | Original: `{self.definition}` | New: `{definition}`",
            )
        self.definition = definition
set_importer(importer=None, nautobot_name='', override=False)

Sets the importer and Nautobot field if not already specified.

If nautobot_name is not provided, the field name is used.

Passing None to nautobot_name indicates that there is custom mapping without a direct relationship to a Nautobot field.

Source code in nautobot_netbox_importer/generator/source.py
def set_importer(
    self,
    importer: Optional[SourceFieldImporter] = None,
    nautobot_name: Optional[FieldName] = "",
    override=False,
) -> Optional[SourceFieldImporter]:
    """Sets the importer and Nautobot field if not already specified.

    If `nautobot_name` is not provided, the field name is used.

    Passing None to `nautobot_name` indicates that there is custom mapping without a direct relationship to a Nautobot field.
    """
    if self.disable_reason:
        raise RuntimeError(f"Can't set importer for disabled {self}")
    if self.importer and not override:
        raise RuntimeError(f"Importer already set for {self}")
    if not self._nautobot and nautobot_name is not None:
        self.set_nautobot_field(nautobot_name)

    if importer:
        self.importer = importer
        return importer

    if self.disable_reason or not self.nautobot.can_import:
        return None

    internal_type = self.nautobot.internal_type

    if internal_type == InternalFieldType.JSON_FIELD:
        self.set_json_importer()
    elif internal_type == InternalFieldType.DATE_FIELD:
        self.set_date_importer()
    elif internal_type == InternalFieldType.DATE_TIME_FIELD:
        self.set_datetime_importer()
    elif internal_type == InternalFieldType.UUID_FIELD:
        self.set_uuid_importer()
    elif internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
        self.set_m2m_importer()
    elif internal_type == InternalFieldType.STATUS_FIELD:
        self.set_status_importer()
    elif self.nautobot.is_reference:
        self.set_relation_importer()
    elif getattr(self.nautobot.field, "choices", None):
        self.set_choice_importer()
    elif self.nautobot.is_integer:
        self.set_integer_importer()
    else:
        self.set_value_importer()

    return self.importer
set_integer_importer()

Set an integer field importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_integer_importer(self) -> None:
    """Set an integer field importer."""

    def integer_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        source_value = self.get_source_value(source)
        if source_value in EMPTY_VALUES:
            self.set_nautobot_value(target, source_value)
        else:
            source_value = float(source_value)
            value = int(source_value)
            self.set_nautobot_value(target, value)
            if value != source_value:
                raise SourceFieldImporterIssue(f"Invalid source value {source_value}, truncated to {value}", self)

    self.set_importer(integer_importer)
set_json_importer()

Set a JSON field importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_json_importer(self) -> None:
    """Set a JSON field importer."""

    def json_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = source.get(self.name, None)
        if isinstance(value, str) and value:
            value = json.loads(value)
        self.set_nautobot_value(target, value)

    self.set_importer(json_importer)
set_m2m_importer(related_source=None)

Set a many to many importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_m2m_importer(self, related_source: Optional[SourceContentType] = None) -> None:
    """Set a many to many importer."""
    if not isinstance(self.nautobot.field, DjangoField):
        raise NotImplementedError(f"Unsupported m2m importer {self}")

    related_wrapper = self._get_related_wrapper(related_source)

    if related_wrapper.content_type == "contenttypes.contenttype":
        self.set_content_types_importer()
        return

    def m2m_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        values = source.get(self.name, None)
        if values in EMPTY_VALUES:
            return

        if isinstance(values, (UUID, str, int)):
            result = related_wrapper.get_pk_from_uid(values)
            self.wrapper.add_reference(related_wrapper, result)
            self.set_nautobot_value(target, {result})
            return

        if not isinstance(values, (list, set)):
            raise ValueError(f"Invalid value {values} for field {self.name}")

        results = set()
        for value in values:
            if isinstance(value, (UUID, str, int)):
                result = related_wrapper.get_pk_from_uid(value)
            else:
                result = related_wrapper.get_pk_from_identifiers(value)

            results.add(result)
            self.wrapper.add_reference(related_wrapper, result)

        self.set_nautobot_value(target, results)

    self.set_importer(m2m_importer)
set_nautobot_field(nautobot_name='')

Set a Nautobot field name for the field.

Source code in nautobot_netbox_importer/generator/source.py
def set_nautobot_field(self, nautobot_name: FieldName = "") -> NautobotField:
    """Set a Nautobot field name for the field."""
    result = self.wrapper.nautobot.add_field(nautobot_name or self.name)
    if result.field:
        default_value = getattr(result.field, "default", None)
        if default_value not in EMPTY_VALUES and not isinstance(default_value, Callable):
            self.default_value = default_value
    self._nautobot = result
    if result.name == "last_updated":
        self.disable("Last updated field is updated with each write")
    return result
set_nautobot_value(target, value)

Set a value to the Nautobot model.

Source code in nautobot_netbox_importer/generator/source.py
def set_nautobot_value(self, target: DiffSyncModel, value: Any) -> None:
    """Set a value to the Nautobot model."""
    if value in EMPTY_VALUES:
        if hasattr(target, self.nautobot.name):
            delattr(target, self.nautobot.name)
    else:
        setattr(target, self.nautobot.name, value)
set_relation_and_type_importer(type_field)

Set a relation UUID importer based on the type field.

Source code in nautobot_netbox_importer/generator/source.py
def set_relation_and_type_importer(self, type_field: "SourceField") -> None:
    """Set a relation UUID importer based on the type field."""

    def relation_and_type_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        source_uid = source.get(self.name, None)
        source_type = source.get(type_field.name, None)
        if source_type in EMPTY_VALUES or source_uid in EMPTY_VALUES:
            if source_uid not in EMPTY_VALUES or source_type not in EMPTY_VALUES:
                raise ValueError(
                    f"Both {self}=`{source_uid}` and {type_field}=`{source_type}` must be empty or not empty."
                )
            return

        type_wrapper = self.wrapper.adapter.get_or_create_wrapper(source_type)
        uid = type_wrapper.get_pk_from_uid(source_uid)
        self.set_nautobot_value(target, uid)
        type_field.set_nautobot_value(target, type_wrapper.nautobot.content_type_instance.pk)
        self.wrapper.add_reference(type_wrapper, uid)

    self.set_importer(relation_and_type_importer)
    self.handle_sibling(type_field, type_field.name)
set_relation_importer(related_source=None)

Set a relation importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_relation_importer(self, related_source: Optional[SourceContentType] = None) -> None:
    """Set a relation importer."""
    related_wrapper = self._get_related_wrapper(related_source)

    if self.nautobot.is_content_type:
        self.set_content_type_importer()
        return

    if self.default_value in EMPTY_VALUES and related_wrapper.default_reference_uid:
        self.default_value = related_wrapper.default_reference_uid

    if not (self.default_value is None or isinstance(self.default_value, UUID)):
        raise NotImplementedError(f"Default value {self.default_value} is not a UUID")

    def relation_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = self.get_source_value(source)
        if value in EMPTY_VALUES:
            self.set_nautobot_value(target, value)
        else:
            if isinstance(value, (UUID, str, int)):
                result = related_wrapper.get_pk_from_uid(value)
            else:
                result = related_wrapper.get_pk_from_identifiers(value)
            self.set_nautobot_value(target, result)
            self.wrapper.add_reference(related_wrapper, result)

    self.set_importer(relation_importer)
set_status_importer()

Set a status importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_status_importer(self) -> None:
    """Set a status importer."""
    status_wrapper = self.wrapper.adapter.get_or_create_wrapper("extras.status")
    if not self.default_value:
        self.default_value = status_wrapper.default_reference_uid

    if not (self.default_value is None or isinstance(self.default_value, UUID)):
        raise NotImplementedError(f"Default value {self.default_value} is not a UUID")

    def status_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        status = source.get(self.name, None)
        if status:
            value = status_wrapper.cache_record({"name": status[0].upper() + status[1:]})
        else:
            value = self.default_value

        self.set_nautobot_value(target, value)
        if value:
            self.wrapper.add_reference(status_wrapper, value)

    self.set_importer(status_importer)
set_uuid_importer()

Set an UUID importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_uuid_importer(self) -> None:
    """Set an UUID importer."""
    if self.name.endswith("_id"):
        type_field = self.wrapper.fields.get(self.name[:-3] + "_type", None)
        if type_field and type_field.nautobot.is_content_type:
            # Handles `<field name>_id` and `<field name>_type` fields combination
            self.set_relation_and_type_importer(type_field)
            return

    def uuid_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = source.get(self.name, None)
        if value not in EMPTY_VALUES:
            value = UUID(value)
        self.set_nautobot_value(target, value)

    self.set_importer(uuid_importer)
set_value_importer()

Set a value importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_value_importer(self) -> None:
    """Set a value importer."""

    def value_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = self.get_source_value(source)
        self.set_nautobot_value(target, value)

    self.set_importer(value_importer)

SourceFieldImporterIssue

Bases: NetBoxImporterException

Raised when an error occurs during field import.

Source code in nautobot_netbox_importer/generator/source.py
class SourceFieldImporterIssue(NetBoxImporterException):
    """Raised when an error occurs during field import."""

    def __init__(self, message: str, field: "SourceField"):
        """Initialize the exception."""
        super().__init__(str({field.name: message}))
__init__(message, field)

Initialize the exception.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(self, message: str, field: "SourceField"):
    """Initialize the exception."""
    super().__init__(str({field.name: message}))

SourceModelWrapper

Definition of a source model mapping to Nautobot model.

Source code in nautobot_netbox_importer/generator/source.py
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
class SourceModelWrapper:
    """Definition of a source model mapping to Nautobot model."""

    def __init__(self, adapter: SourceAdapter, content_type: ContentTypeStr, nautobot_wrapper: NautobotModelWrapper):
        """Initialize the SourceModelWrapper."""
        if content_type in adapter.wrappers:
            raise ValueError(f"Duplicate content type {content_type}")
        adapter.wrappers[content_type] = self
        self.adapter = adapter
        self.content_type = content_type
        self.nautobot = nautobot_wrapper
        if self.nautobot.disabled:
            self.disable_reason = f"Nautobot content type: `{nautobot_wrapper.content_type}` not found"
        else:
            self.disable_reason = ""

        # Source field names when referencing this model
        self.identifiers: Optional[List[FieldName]] = None

        # Used to autofill `content_types` field
        self.disable_related_reference = False
        self.references: SourceReferences = {}
        self.forward_references: Optional[ForwardReferences] = None

        # Whether importing record data exteds existing record
        self.extends_wrapper: Optional[SourceModelWrapper] = None

        # Importers are created after all fields are defined
        self.importers: Optional[Set[SourceFieldImporter]] = None

        # Default reference to this model
        self.default_reference_uid: Optional[Uid] = None

        # Caching
        self._uid_to_pk_cache: Dict[Uid, Uid] = {}
        self._cached_data: Dict[Uid, RecordData] = {}

        self.stats = SourceModelStats()
        self.flags = DiffSyncModelFlags.NONE

        # Source fields defintions
        self.fields: OrderedDict[FieldName, SourceField] = OrderedDict()
        self.pre_import: Optional[PreImport] = None

        if self.disable_reason:
            self.adapter.logger.debug("Created disabled %s", self)
            return

        pk_field = self.add_field(nautobot_wrapper.pk_field.name, SourceFieldSource.AUTO)
        pk_field.set_nautobot_field()
        pk_field.processed = True

        if issubclass(nautobot_wrapper.model, TreeModel):
            for name in ("tree_id", "lft", "rght", "level"):
                self.disable_field(name, "Tree fields doesn't need to be imported")

        self.adapter.logger.debug("Created %s", self)

    def __str__(self) -> str:
        """Return a string representation of the wrapper."""
        return f"{self.__class__.__name__}<{self.content_type} -> {self.nautobot.content_type}>"

    def cache_record_uids(self, source: RecordData, nautobot_uid: Optional[Uid] = None) -> Uid:
        """Cache record identifier mappings.

        When `nautobot_uid` is not provided, it is generated from the source data and caching is processed there.
        """
        if not nautobot_uid:
            return self.get_pk_from_data(source)

        if self.identifiers:
            identifiers_data = [source[field_name] for field_name in self.identifiers]
            self._uid_to_pk_cache[json.dumps(identifiers_data)] = nautobot_uid

        source_uid = source.get(self.nautobot.pk_field.name, None)
        if source_uid and source_uid not in self._uid_to_pk_cache:
            self._uid_to_pk_cache[source_uid] = nautobot_uid

        self._uid_to_pk_cache[nautobot_uid] = nautobot_uid

        return nautobot_uid

    def first_pass(self, data: RecordData) -> None:
        """Firts pass of data import."""
        if self.pre_import:
            if self.pre_import(data, ImporterPass.DEFINE_STRUCTURE) != PreImportResult.USE_RECORD:
                self.stats.first_pass_skipped += 1
                return

        self.stats.first_pass_used += 1

        if self.disable_reason:
            return

        for field_name in data.keys():
            self.add_field(field_name, SourceFieldSource.DATA)

    def second_pass(self, data: RecordData) -> None:
        """Second pass of data import."""
        if self.disable_reason:
            return

        if self.pre_import:
            if self.pre_import(data, ImporterPass.IMPORT_DATA) != PreImportResult.USE_RECORD:
                self.stats.second_pass_skipped += 1
                return

        self.stats.second_pass_used += 1

        self.import_record(data)

    def get_summary(self, content_type_id) -> SourceModelSummary:
        """Get a summary of the model."""
        fields = [field.get_summary() for field in self.fields.values()]

        return SourceModelSummary(
            content_type=self.content_type,
            content_type_id=content_type_id,
            extends_content_type=self.extends_wrapper and self.extends_wrapper.content_type,
            nautobot_content_type=self.nautobot.content_type,
            disable_reason=self.disable_reason,
            identifiers=self.identifiers,
            disable_related_reference=self.disable_related_reference,
            forward_references=self.forward_references and self.forward_references.__name__ or None,
            pre_import=self.pre_import and self.pre_import.__name__ or None,
            fields=sorted(fields, key=lambda field: field.name),
            flags=str(self.flags),
            default_reference_uid=serialize_to_summary(self.default_reference_uid),
            stats=self.stats,
        )

    def set_identifiers(self, identifiers: Iterable[FieldName]) -> None:
        """Set identifiers for the model."""
        if self.identifiers:
            if list(identifiers) == self.identifiers:
                return
            raise ValueError(
                f"Different identifiers were already set up | original: `{self.identifiers}` | new: `{identifiers}`"
            )

        if list(identifiers) == [self.nautobot.pk_field.name]:
            return

        self.identifiers = list(identifiers)
        for identifier in self.identifiers:
            self.add_field(identifier, SourceFieldSource.IDENTIFIER)

    def disable_field(self, field_name: FieldName, reason: str) -> "SourceField":
        """Disable field importing."""
        field = self.add_field(field_name, SourceFieldSource.CUSTOM)
        field.disable(reason)
        return field

    def format_field_name(self, name: FieldName) -> str:
        """Format a field name for logging."""
        return f"{self.content_type}->{name}"

    def add_field(self, name: FieldName, source: SourceFieldSource) -> "SourceField":
        """Add a field definition for a source field."""
        if self.importers is not None:
            raise ValueError(f"Can't add field {self.format_field_name(name)}, model's importers already created.")

        if name not in self.fields:
            return SourceField(self, name, source)

        field = self.fields[name]
        field.sources.add(source)
        return field

    def create_importers(self) -> None:
        """Create importers for all fields."""
        if self.importers is not None:
            raise RuntimeError(f"Importers already created for {self.content_type}")

        if not self.extends_wrapper:
            for field_name in AUTO_ADD_FIELDS:
                if hasattr(self.nautobot.model, field_name):
                    self.add_field(field_name, SourceFieldSource.AUTO)

        while True:
            fields = [field for field in self.fields.values() if not field.processed]
            if not fields:
                break

            for field in fields:
                try:
                    field.create_importer()
                except Exception:
                    self.adapter.logger.error("Failed to create importer for %s", field)
                    raise

        self.importers = set(field.importer for field in self.fields.values() if field.importer)

    def get_pk_from_uid(self, uid: Uid) -> Uid:
        """Get a source primary key for a given source uid."""
        if uid in self._uid_to_pk_cache:
            return self._uid_to_pk_cache[uid]

        if self.nautobot.pk_field.internal_type == InternalFieldType.UUID_FIELD:
            if self.extends_wrapper:
                result = self.extends_wrapper.get_pk_from_uid(uid)
            else:
                result = source_pk_to_uuid(self.content_type or self.content_type, uid)
        elif self.nautobot.pk_field.is_auto_increment:
            self.nautobot.last_id += 1
            result = self.nautobot.last_id
        else:
            raise ValueError(f"Unsupported pk_type {self.nautobot.pk_field.internal_type}")

        self._uid_to_pk_cache[uid] = result
        self._uid_to_pk_cache[result] = result

        return result

    def get_pk_from_identifiers(self, data: Union[Uid, Iterable[Uid]]) -> Uid:
        """Get a source primary key for a given source identifiers."""
        if not self.identifiers:
            if isinstance(data, (UUID, str, int)):
                return self.get_pk_from_uid(data)

            raise ValueError(f"Invalid identifiers {data} for {self.identifiers}")

        if not isinstance(data, list):
            data = list(data)  # type: ignore
        if len(self.identifiers) != len(data):
            raise ValueError(f"Invalid identifiers {data} for {self.identifiers}")

        identifiers_uid = json.dumps(data)
        if identifiers_uid in self._uid_to_pk_cache:
            return self._uid_to_pk_cache[identifiers_uid]

        filter_kwargs = {self.identifiers[index]: value for index, value in enumerate(data)}
        try:
            nautobot_instance = self.nautobot.model.objects.get(**filter_kwargs)
            nautobot_uid = getattr(nautobot_instance, self.nautobot.pk_field.name)
            if not nautobot_uid:
                raise ValueError(f"Invalid args {filter_kwargs} for {nautobot_instance}")
            self._uid_to_pk_cache[identifiers_uid] = nautobot_uid
            self._uid_to_pk_cache[nautobot_uid] = nautobot_uid
            return nautobot_uid
        except self.nautobot.model.DoesNotExist:  # type: ignore
            return self.get_pk_from_uid(identifiers_uid)

    def get_pk_from_data(self, data: RecordData) -> Uid:
        """Get a source primary key for a given source data."""
        if not self.identifiers:
            return self.get_pk_from_uid(data[self.nautobot.pk_field.name])

        data_uid = data.get(self.nautobot.pk_field.name, None)
        if data_uid and data_uid in self._uid_to_pk_cache:
            return self._uid_to_pk_cache[data_uid]

        result = self.get_pk_from_identifiers(data[field_name] for field_name in self.identifiers)

        if data_uid:
            self._uid_to_pk_cache[data_uid] = result

        return result

    def import_record(self, data: RecordData, target: Optional[DiffSyncBaseModel] = None) -> DiffSyncBaseModel:
        """Import a single item from the source."""
        self.adapter.logger.debug("Importing record %s %s", self, data)
        if self.importers is None:
            raise RuntimeError(f"Importers not created for {self}")

        if target:
            uid = getattr(target, self.nautobot.pk_field.name)
        else:
            uid = self.get_pk_from_data(data)
            target = self.get_or_create(uid)

        for importer in self.importers:
            try:
                importer(data, target)
            # pylint: disable=broad-exception-caught
            except Exception as error:
                field = next(field for field in self.fields.values() if field.importer == importer)
                self.nautobot.add_issue(
                    diffsync_instance=target,
                    error=error,
                    message=str({"field": field.name if field else None, "importer": importer.__name__}),
                )

        self.stats.imported += 1
        self.adapter.logger.debug("Imported %s %s", uid, target.get_attrs())

        return target

    def get_or_create(self, uid: Uid, fail_missing=False) -> DiffSyncBaseModel:
        """Get an existing DiffSync Model instance from the source or create a new one.

        Use Nautobot data as defaults if available.
        """
        filter_kwargs = {self.nautobot.pk_field.name: uid}
        diffsync_class = self.nautobot.diffsync_class
        result = self.adapter.get_or_none(diffsync_class, filter_kwargs)
        if result:
            if not isinstance(result, DiffSyncBaseModel):
                raise TypeError(f"Invalid instance type {result}")
            return result

        result = diffsync_class(**filter_kwargs, diffsync=self.adapter)  # type: ignore
        result.model_flags = self.flags

        cached_data = self._cached_data.get(uid, None)
        if cached_data:
            fail_missing = False
            self.import_record(cached_data, result)
            self.stats.imported_from_cache += 1

        nautobot_diffsync_instance = self.nautobot.find_or_create(filter_kwargs)
        if nautobot_diffsync_instance:
            fail_missing = False
            for key, value in nautobot_diffsync_instance.get_attrs().items():
                if value not in EMPTY_VALUES:
                    setattr(result, key, value)

        if fail_missing:
            raise ValueError(f"Missing {self} {uid} in Nautobot or cached data")

        self.adapter.add(result)
        self.stats.created += 1
        if self.flags == DiffSyncModelFlags.IGNORE:
            self.nautobot.stats.source_ignored += 1
        else:
            self.nautobot.stats.source_created += 1

        return result

    def get_default_reference_uid(self) -> Uid:
        """Get the default reference to this model."""
        if self.default_reference_uid:
            return self.default_reference_uid
        raise ValueError("Missing default reference")

    def cache_record(self, data: RecordData) -> Uid:
        """Cache data for optional later use.

        If record is referenced by other models, it will be imported automatically; otherwise, it will be ignored.
        """
        uid = self.get_pk_from_data(data)
        if uid in self._cached_data:
            return uid

        if self.importers is None:
            for field_name in data.keys():
                self.add_field(field_name, SourceFieldSource.CACHE)

        self._cached_data[uid] = data
        self.stats.pre_cached += 1

        self.adapter.logger.debug("Cached %s %s %s", self, uid, data)

        return uid

    def set_default_reference(self, data: RecordData) -> None:
        """Set the default reference to this model."""
        self.default_reference_uid = self.cache_record(data)

    def post_import(self) -> bool:
        """Post import processing.

        Assigns referenced content_types to referencing instances.

        Returns False if no post processing is needed, otherwise True to indicate that post processing is needed.
        """
        if not self.references:
            return False

        references = self.references
        self.references = {}

        if self.forward_references:
            self.forward_references(self, references)
            return True

        for uid, content_types in references.items():
            # Keep this even when no content_types field is present, to create referenced cached data
            instance = self.get_or_create(uid, fail_missing=True)
            if "content_types" not in self.nautobot.fields:
                continue

            content_types = set(wrapper.nautobot.content_type_instance.pk for wrapper in content_types)
            target_content_types = getattr(instance, "content_types", None)
            if target_content_types != content_types:
                if target_content_types:
                    target_content_types.update(content_types)
                else:
                    instance.content_types = content_types
                self.adapter.update(instance)

        return True

    def add_reference(self, related_wrapper: "SourceModelWrapper", uid: Uid) -> None:
        """Add a reference from this content type to related record."""
        if self.disable_related_reference:
            return
        self.adapter.logger.debug(
            "Adding reference from: %s to: %s %s", self.content_type, related_wrapper.content_type, uid
        )
        if not uid:
            raise ValueError(f"Invalid uid {uid}")
        related_wrapper.references.setdefault(uid, set()).add(self)
__init__(adapter, content_type, nautobot_wrapper)

Initialize the SourceModelWrapper.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(self, adapter: SourceAdapter, content_type: ContentTypeStr, nautobot_wrapper: NautobotModelWrapper):
    """Initialize the SourceModelWrapper."""
    if content_type in adapter.wrappers:
        raise ValueError(f"Duplicate content type {content_type}")
    adapter.wrappers[content_type] = self
    self.adapter = adapter
    self.content_type = content_type
    self.nautobot = nautobot_wrapper
    if self.nautobot.disabled:
        self.disable_reason = f"Nautobot content type: `{nautobot_wrapper.content_type}` not found"
    else:
        self.disable_reason = ""

    # Source field names when referencing this model
    self.identifiers: Optional[List[FieldName]] = None

    # Used to autofill `content_types` field
    self.disable_related_reference = False
    self.references: SourceReferences = {}
    self.forward_references: Optional[ForwardReferences] = None

    # Whether importing record data exteds existing record
    self.extends_wrapper: Optional[SourceModelWrapper] = None

    # Importers are created after all fields are defined
    self.importers: Optional[Set[SourceFieldImporter]] = None

    # Default reference to this model
    self.default_reference_uid: Optional[Uid] = None

    # Caching
    self._uid_to_pk_cache: Dict[Uid, Uid] = {}
    self._cached_data: Dict[Uid, RecordData] = {}

    self.stats = SourceModelStats()
    self.flags = DiffSyncModelFlags.NONE

    # Source fields defintions
    self.fields: OrderedDict[FieldName, SourceField] = OrderedDict()
    self.pre_import: Optional[PreImport] = None

    if self.disable_reason:
        self.adapter.logger.debug("Created disabled %s", self)
        return

    pk_field = self.add_field(nautobot_wrapper.pk_field.name, SourceFieldSource.AUTO)
    pk_field.set_nautobot_field()
    pk_field.processed = True

    if issubclass(nautobot_wrapper.model, TreeModel):
        for name in ("tree_id", "lft", "rght", "level"):
            self.disable_field(name, "Tree fields doesn't need to be imported")

    self.adapter.logger.debug("Created %s", self)
__str__()

Return a string representation of the wrapper.

Source code in nautobot_netbox_importer/generator/source.py
def __str__(self) -> str:
    """Return a string representation of the wrapper."""
    return f"{self.__class__.__name__}<{self.content_type} -> {self.nautobot.content_type}>"
add_field(name, source)

Add a field definition for a source field.

Source code in nautobot_netbox_importer/generator/source.py
def add_field(self, name: FieldName, source: SourceFieldSource) -> "SourceField":
    """Add a field definition for a source field."""
    if self.importers is not None:
        raise ValueError(f"Can't add field {self.format_field_name(name)}, model's importers already created.")

    if name not in self.fields:
        return SourceField(self, name, source)

    field = self.fields[name]
    field.sources.add(source)
    return field
add_reference(related_wrapper, uid)

Add a reference from this content type to related record.

Source code in nautobot_netbox_importer/generator/source.py
def add_reference(self, related_wrapper: "SourceModelWrapper", uid: Uid) -> None:
    """Add a reference from this content type to related record."""
    if self.disable_related_reference:
        return
    self.adapter.logger.debug(
        "Adding reference from: %s to: %s %s", self.content_type, related_wrapper.content_type, uid
    )
    if not uid:
        raise ValueError(f"Invalid uid {uid}")
    related_wrapper.references.setdefault(uid, set()).add(self)
cache_record(data)

Cache data for optional later use.

If record is referenced by other models, it will be imported automatically; otherwise, it will be ignored.

Source code in nautobot_netbox_importer/generator/source.py
def cache_record(self, data: RecordData) -> Uid:
    """Cache data for optional later use.

    If record is referenced by other models, it will be imported automatically; otherwise, it will be ignored.
    """
    uid = self.get_pk_from_data(data)
    if uid in self._cached_data:
        return uid

    if self.importers is None:
        for field_name in data.keys():
            self.add_field(field_name, SourceFieldSource.CACHE)

    self._cached_data[uid] = data
    self.stats.pre_cached += 1

    self.adapter.logger.debug("Cached %s %s %s", self, uid, data)

    return uid
cache_record_uids(source, nautobot_uid=None)

Cache record identifier mappings.

When nautobot_uid is not provided, it is generated from the source data and caching is processed there.

Source code in nautobot_netbox_importer/generator/source.py
def cache_record_uids(self, source: RecordData, nautobot_uid: Optional[Uid] = None) -> Uid:
    """Cache record identifier mappings.

    When `nautobot_uid` is not provided, it is generated from the source data and caching is processed there.
    """
    if not nautobot_uid:
        return self.get_pk_from_data(source)

    if self.identifiers:
        identifiers_data = [source[field_name] for field_name in self.identifiers]
        self._uid_to_pk_cache[json.dumps(identifiers_data)] = nautobot_uid

    source_uid = source.get(self.nautobot.pk_field.name, None)
    if source_uid and source_uid not in self._uid_to_pk_cache:
        self._uid_to_pk_cache[source_uid] = nautobot_uid

    self._uid_to_pk_cache[nautobot_uid] = nautobot_uid

    return nautobot_uid
create_importers()

Create importers for all fields.

Source code in nautobot_netbox_importer/generator/source.py
def create_importers(self) -> None:
    """Create importers for all fields."""
    if self.importers is not None:
        raise RuntimeError(f"Importers already created for {self.content_type}")

    if not self.extends_wrapper:
        for field_name in AUTO_ADD_FIELDS:
            if hasattr(self.nautobot.model, field_name):
                self.add_field(field_name, SourceFieldSource.AUTO)

    while True:
        fields = [field for field in self.fields.values() if not field.processed]
        if not fields:
            break

        for field in fields:
            try:
                field.create_importer()
            except Exception:
                self.adapter.logger.error("Failed to create importer for %s", field)
                raise

    self.importers = set(field.importer for field in self.fields.values() if field.importer)
disable_field(field_name, reason)

Disable field importing.

Source code in nautobot_netbox_importer/generator/source.py
def disable_field(self, field_name: FieldName, reason: str) -> "SourceField":
    """Disable field importing."""
    field = self.add_field(field_name, SourceFieldSource.CUSTOM)
    field.disable(reason)
    return field
first_pass(data)

Firts pass of data import.

Source code in nautobot_netbox_importer/generator/source.py
def first_pass(self, data: RecordData) -> None:
    """Firts pass of data import."""
    if self.pre_import:
        if self.pre_import(data, ImporterPass.DEFINE_STRUCTURE) != PreImportResult.USE_RECORD:
            self.stats.first_pass_skipped += 1
            return

    self.stats.first_pass_used += 1

    if self.disable_reason:
        return

    for field_name in data.keys():
        self.add_field(field_name, SourceFieldSource.DATA)
format_field_name(name)

Format a field name for logging.

Source code in nautobot_netbox_importer/generator/source.py
def format_field_name(self, name: FieldName) -> str:
    """Format a field name for logging."""
    return f"{self.content_type}->{name}"
get_default_reference_uid()

Get the default reference to this model.

Source code in nautobot_netbox_importer/generator/source.py
def get_default_reference_uid(self) -> Uid:
    """Get the default reference to this model."""
    if self.default_reference_uid:
        return self.default_reference_uid
    raise ValueError("Missing default reference")
get_or_create(uid, fail_missing=False)

Get an existing DiffSync Model instance from the source or create a new one.

Use Nautobot data as defaults if available.

Source code in nautobot_netbox_importer/generator/source.py
def get_or_create(self, uid: Uid, fail_missing=False) -> DiffSyncBaseModel:
    """Get an existing DiffSync Model instance from the source or create a new one.

    Use Nautobot data as defaults if available.
    """
    filter_kwargs = {self.nautobot.pk_field.name: uid}
    diffsync_class = self.nautobot.diffsync_class
    result = self.adapter.get_or_none(diffsync_class, filter_kwargs)
    if result:
        if not isinstance(result, DiffSyncBaseModel):
            raise TypeError(f"Invalid instance type {result}")
        return result

    result = diffsync_class(**filter_kwargs, diffsync=self.adapter)  # type: ignore
    result.model_flags = self.flags

    cached_data = self._cached_data.get(uid, None)
    if cached_data:
        fail_missing = False
        self.import_record(cached_data, result)
        self.stats.imported_from_cache += 1

    nautobot_diffsync_instance = self.nautobot.find_or_create(filter_kwargs)
    if nautobot_diffsync_instance:
        fail_missing = False
        for key, value in nautobot_diffsync_instance.get_attrs().items():
            if value not in EMPTY_VALUES:
                setattr(result, key, value)

    if fail_missing:
        raise ValueError(f"Missing {self} {uid} in Nautobot or cached data")

    self.adapter.add(result)
    self.stats.created += 1
    if self.flags == DiffSyncModelFlags.IGNORE:
        self.nautobot.stats.source_ignored += 1
    else:
        self.nautobot.stats.source_created += 1

    return result
get_pk_from_data(data)

Get a source primary key for a given source data.

Source code in nautobot_netbox_importer/generator/source.py
def get_pk_from_data(self, data: RecordData) -> Uid:
    """Get a source primary key for a given source data."""
    if not self.identifiers:
        return self.get_pk_from_uid(data[self.nautobot.pk_field.name])

    data_uid = data.get(self.nautobot.pk_field.name, None)
    if data_uid and data_uid in self._uid_to_pk_cache:
        return self._uid_to_pk_cache[data_uid]

    result = self.get_pk_from_identifiers(data[field_name] for field_name in self.identifiers)

    if data_uid:
        self._uid_to_pk_cache[data_uid] = result

    return result
get_pk_from_identifiers(data)

Get a source primary key for a given source identifiers.

Source code in nautobot_netbox_importer/generator/source.py
def get_pk_from_identifiers(self, data: Union[Uid, Iterable[Uid]]) -> Uid:
    """Get a source primary key for a given source identifiers."""
    if not self.identifiers:
        if isinstance(data, (UUID, str, int)):
            return self.get_pk_from_uid(data)

        raise ValueError(f"Invalid identifiers {data} for {self.identifiers}")

    if not isinstance(data, list):
        data = list(data)  # type: ignore
    if len(self.identifiers) != len(data):
        raise ValueError(f"Invalid identifiers {data} for {self.identifiers}")

    identifiers_uid = json.dumps(data)
    if identifiers_uid in self._uid_to_pk_cache:
        return self._uid_to_pk_cache[identifiers_uid]

    filter_kwargs = {self.identifiers[index]: value for index, value in enumerate(data)}
    try:
        nautobot_instance = self.nautobot.model.objects.get(**filter_kwargs)
        nautobot_uid = getattr(nautobot_instance, self.nautobot.pk_field.name)
        if not nautobot_uid:
            raise ValueError(f"Invalid args {filter_kwargs} for {nautobot_instance}")
        self._uid_to_pk_cache[identifiers_uid] = nautobot_uid
        self._uid_to_pk_cache[nautobot_uid] = nautobot_uid
        return nautobot_uid
    except self.nautobot.model.DoesNotExist:  # type: ignore
        return self.get_pk_from_uid(identifiers_uid)
get_pk_from_uid(uid)

Get a source primary key for a given source uid.

Source code in nautobot_netbox_importer/generator/source.py
def get_pk_from_uid(self, uid: Uid) -> Uid:
    """Get a source primary key for a given source uid."""
    if uid in self._uid_to_pk_cache:
        return self._uid_to_pk_cache[uid]

    if self.nautobot.pk_field.internal_type == InternalFieldType.UUID_FIELD:
        if self.extends_wrapper:
            result = self.extends_wrapper.get_pk_from_uid(uid)
        else:
            result = source_pk_to_uuid(self.content_type or self.content_type, uid)
    elif self.nautobot.pk_field.is_auto_increment:
        self.nautobot.last_id += 1
        result = self.nautobot.last_id
    else:
        raise ValueError(f"Unsupported pk_type {self.nautobot.pk_field.internal_type}")

    self._uid_to_pk_cache[uid] = result
    self._uid_to_pk_cache[result] = result

    return result
get_summary(content_type_id)

Get a summary of the model.

Source code in nautobot_netbox_importer/generator/source.py
def get_summary(self, content_type_id) -> SourceModelSummary:
    """Get a summary of the model."""
    fields = [field.get_summary() for field in self.fields.values()]

    return SourceModelSummary(
        content_type=self.content_type,
        content_type_id=content_type_id,
        extends_content_type=self.extends_wrapper and self.extends_wrapper.content_type,
        nautobot_content_type=self.nautobot.content_type,
        disable_reason=self.disable_reason,
        identifiers=self.identifiers,
        disable_related_reference=self.disable_related_reference,
        forward_references=self.forward_references and self.forward_references.__name__ or None,
        pre_import=self.pre_import and self.pre_import.__name__ or None,
        fields=sorted(fields, key=lambda field: field.name),
        flags=str(self.flags),
        default_reference_uid=serialize_to_summary(self.default_reference_uid),
        stats=self.stats,
    )
import_record(data, target=None)

Import a single item from the source.

Source code in nautobot_netbox_importer/generator/source.py
def import_record(self, data: RecordData, target: Optional[DiffSyncBaseModel] = None) -> DiffSyncBaseModel:
    """Import a single item from the source."""
    self.adapter.logger.debug("Importing record %s %s", self, data)
    if self.importers is None:
        raise RuntimeError(f"Importers not created for {self}")

    if target:
        uid = getattr(target, self.nautobot.pk_field.name)
    else:
        uid = self.get_pk_from_data(data)
        target = self.get_or_create(uid)

    for importer in self.importers:
        try:
            importer(data, target)
        # pylint: disable=broad-exception-caught
        except Exception as error:
            field = next(field for field in self.fields.values() if field.importer == importer)
            self.nautobot.add_issue(
                diffsync_instance=target,
                error=error,
                message=str({"field": field.name if field else None, "importer": importer.__name__}),
            )

    self.stats.imported += 1
    self.adapter.logger.debug("Imported %s %s", uid, target.get_attrs())

    return target
post_import()

Post import processing.

Assigns referenced content_types to referencing instances.

Returns False if no post processing is needed, otherwise True to indicate that post processing is needed.

Source code in nautobot_netbox_importer/generator/source.py
def post_import(self) -> bool:
    """Post import processing.

    Assigns referenced content_types to referencing instances.

    Returns False if no post processing is needed, otherwise True to indicate that post processing is needed.
    """
    if not self.references:
        return False

    references = self.references
    self.references = {}

    if self.forward_references:
        self.forward_references(self, references)
        return True

    for uid, content_types in references.items():
        # Keep this even when no content_types field is present, to create referenced cached data
        instance = self.get_or_create(uid, fail_missing=True)
        if "content_types" not in self.nautobot.fields:
            continue

        content_types = set(wrapper.nautobot.content_type_instance.pk for wrapper in content_types)
        target_content_types = getattr(instance, "content_types", None)
        if target_content_types != content_types:
            if target_content_types:
                target_content_types.update(content_types)
            else:
                instance.content_types = content_types
            self.adapter.update(instance)

    return True
second_pass(data)

Second pass of data import.

Source code in nautobot_netbox_importer/generator/source.py
def second_pass(self, data: RecordData) -> None:
    """Second pass of data import."""
    if self.disable_reason:
        return

    if self.pre_import:
        if self.pre_import(data, ImporterPass.IMPORT_DATA) != PreImportResult.USE_RECORD:
            self.stats.second_pass_skipped += 1
            return

    self.stats.second_pass_used += 1

    self.import_record(data)
set_default_reference(data)

Set the default reference to this model.

Source code in nautobot_netbox_importer/generator/source.py
def set_default_reference(self, data: RecordData) -> None:
    """Set the default reference to this model."""
    self.default_reference_uid = self.cache_record(data)
set_identifiers(identifiers)

Set identifiers for the model.

Source code in nautobot_netbox_importer/generator/source.py
def set_identifiers(self, identifiers: Iterable[FieldName]) -> None:
    """Set identifiers for the model."""
    if self.identifiers:
        if list(identifiers) == self.identifiers:
            return
        raise ValueError(
            f"Different identifiers were already set up | original: `{self.identifiers}` | new: `{identifiers}`"
        )

    if list(identifiers) == [self.nautobot.pk_field.name]:
        return

    self.identifiers = list(identifiers)
    for identifier in self.identifiers:
        self.add_field(identifier, SourceFieldSource.IDENTIFIER)

SourceRecord

Bases: NamedTuple

Source Data Item.

Source code in nautobot_netbox_importer/generator/source.py
class SourceRecord(NamedTuple):
    """Source Data Item."""

    content_type: ContentTypeStr
    data: RecordData

base

Generic DiffSync Importer base module.

BaseAdapter

Bases: DiffSync

Base class for Generator Adapters.

Source code in nautobot_netbox_importer/generator/base.py
class BaseAdapter(DiffSync):
    """Base class for Generator Adapters."""

    def __init__(self, *args, **kwargs):
        """Initialize the adapter."""
        super().__init__(*args, **kwargs)

        self.cleanup()

    def cleanup(self):
        """Clean up the adapter."""
        # TBD: Should be fixed in DiffSync.
        # This is a work around to allow testing multiple imports in a single test run.
        self.top_level.clear()
        if isinstance(self.store, LocalStore):
            # pylint: disable=protected-access
            self.store._data.clear()
__init__(*args, **kwargs)

Initialize the adapter.

Source code in nautobot_netbox_importer/generator/base.py
def __init__(self, *args, **kwargs):
    """Initialize the adapter."""
    super().__init__(*args, **kwargs)

    self.cleanup()
cleanup()

Clean up the adapter.

Source code in nautobot_netbox_importer/generator/base.py
def cleanup(self):
    """Clean up the adapter."""
    # TBD: Should be fixed in DiffSync.
    # This is a work around to allow testing multiple imports in a single test run.
    self.top_level.clear()
    if isinstance(self.store, LocalStore):
        # pylint: disable=protected-access
        self.store._data.clear()
InternalFieldType

Bases: Enum

Internal field types.

Source code in nautobot_netbox_importer/generator/base.py
class InternalFieldType(Enum):
    """Internal field types."""

    AUTO_FIELD = "AutoField"
    BIG_AUTO_FIELD = "BigAutoField"
    BIG_INTEGER_FIELD = "BigIntegerField"
    BINARY_FIELD = "BinaryField"
    BOOLEAN_FIELD = "BooleanField"
    CHAR_FIELD = "CharField"
    CUSTOM_FIELD_DATA = "CustomFieldData"
    DATE_FIELD = "DateField"
    DATE_TIME_FIELD = "DateTimeField"
    DECIMAL_FIELD = "DecimalField"
    FOREIGN_KEY = "ForeignKey"
    FOREIGN_KEY_WITH_AUTO_RELATED_NAME = "ForeignKeyWithAutoRelatedName"
    INTEGER_FIELD = "IntegerField"
    JSON_FIELD = "JSONField"
    MANY_TO_MANY_FIELD = "ManyToManyField"
    NOT_FOUND = "NotFound"
    ONE_TO_ONE_FIELD = "OneToOneField"
    POSITIVE_INTEGER_FIELD = "PositiveIntegerField"
    POSITIVE_SMALL_INTEGER_FIELD = "PositiveSmallIntegerField"
    PRIVATE_PROPERTY = "PrivateProperty"
    PROPERTY = "Property"
    READ_ONLY_PROPERTY = "ReadOnlyProperty"
    ROLE_FIELD = "RoleField"
    SLUG_FIELD = "SlugField"
    SMALL_INTEGER_FIELD = "SmallIntegerField"
    STATUS_FIELD = "StatusField"
    TEXT_FIELD = "TextField"
    TREE_NODE_FOREIGN_KEY = "TreeNodeForeignKey"
    UUID_FIELD = "UUIDField"
get_nautobot_field_and_type(model, field_name)

Get Nautobot field and internal field type.

Source code in nautobot_netbox_importer/generator/base.py
def get_nautobot_field_and_type(
    model: NautobotBaseModelType,
    field_name: str,
) -> Tuple[Optional[DjangoField], InternalFieldType]:
    """Get Nautobot field and internal field type."""
    if field_name.startswith("_"):
        return None, InternalFieldType.PRIVATE_PROPERTY

    meta = model._meta  # type: ignore
    if field_name == "custom_field_data":
        field_name = "_custom_field_data"
    try:
        field = meta.get_field(field_name)
    except DjangoFieldDoesNotExist:
        prop = getattr(model, field_name, None)
        if not prop:
            return None, InternalFieldType.NOT_FOUND
        if isinstance(prop, property) and not prop.fset:
            return None, InternalFieldType.READ_ONLY_PROPERTY
        return None, InternalFieldType.PROPERTY

    if field_name == "_custom_field_data":
        return field, InternalFieldType.CUSTOM_FIELD_DATA

    try:
        return field, StrToInternalFieldType[field.get_internal_type()]
    except KeyError as error:
        raise NotImplementedError(f"Unsupported field type {meta.app_label}.{meta.model_name}.{field_name}") from error
normalize_datetime(value)

Normalize datetime values to UTC to compare with DiffSync.

Source code in nautobot_netbox_importer/generator/base.py
def normalize_datetime(value: Any) -> Optional[datetime.datetime]:
    """Normalize datetime values to UTC to compare with DiffSync."""
    if not value:
        return None

    if not isinstance(value, datetime.datetime):
        value = datetime_parser.isoparse(str(value))

    if value.tzinfo is None:
        return value.replace(tzinfo=datetime.timezone.utc)

    return value.astimezone(datetime.timezone.utc)
source_pk_to_uuid(content_type, pk, namespace=uuid5(UUID('33c07af8-e425-43b2-b8d0-52289dfe7cf2'), settings.SECRET_KEY))

Deterministically map source primary key to a UUID primary key.

One of the reasons Nautobot moved from sequential integers to UUIDs was to protect the application against key-enumeration attacks, so we don't use a hard-coded mapping from integer to UUID as that would defeat the purpose.

Source code in nautobot_netbox_importer/generator/base.py
def source_pk_to_uuid(
    content_type: ContentTypeStr,
    pk: Uid,
    # Namespace is defined as random constant UUID combined with settings.SECRET_KEY
    namespace=uuid5(UUID("33c07af8-e425-43b2-b8d0-52289dfe7cf2"), settings.SECRET_KEY),
) -> UUID:
    """Deterministically map source primary key to a UUID primary key.

    One of the reasons Nautobot moved from sequential integers to UUIDs was to protect the application
    against key-enumeration attacks, so we don't use a hard-coded mapping from integer to UUID as that
    would defeat the purpose.
    """
    if isinstance(pk, UUID):
        return pk

    if isinstance(pk, int):
        pk = str(pk)

    if not pk or not isinstance(pk, str):
        raise ValueError(f"Invalid primary key {pk}")

    return uuid5(namespace, f"{content_type}:{pk}")

exceptions

Custom exceptions for the NetBox Importer.

NautobotModelNotFound

Bases: NetBoxImporterException

Raised when a Nautobot model cannot be found.

Source code in nautobot_netbox_importer/generator/exceptions.py
class NautobotModelNotFound(NetBoxImporterException):
    """Raised when a Nautobot model cannot be found."""
NetBoxImporterException

Bases: Exception

Base exception for the netbox_importer package.

Source code in nautobot_netbox_importer/generator/exceptions.py
class NetBoxImporterException(Exception):
    """Base exception for the netbox_importer package."""

fields

Generic Field Importers definitions for Nautobot Importer.

constant(value, nautobot_name='')

Create a constant field definition.

Use to fill target constant value for the field.

Source code in nautobot_netbox_importer/generator/fields.py
def constant(value: Any, nautobot_name: FieldName = "") -> SourceFieldDefinition:
    """Create a constant field definition.

    Use to fill target constant value for the field.
    """

    def define_constant(field: SourceField) -> None:
        def constant_importer(_: RecordData, target: DiffSyncBaseModel) -> None:
            field.set_nautobot_value(target, value)

        field.set_importer(constant_importer, nautobot_name)

    return define_constant
default(default_value, nautobot_name='')

Create a default field definition.

Use to set a default value for the field, if there is no value in the source data.

Source code in nautobot_netbox_importer/generator/fields.py
def default(default_value: Any, nautobot_name: FieldName = "") -> SourceFieldDefinition:
    """Create a default field definition.

    Use to set a default value for the field, if there is no value in the source data.
    """

    def define_default(field: SourceField) -> None:
        field.set_importer(nautobot_name=nautobot_name)
        field.default_value = default_value

    return define_default
disable(reason)

Disable the field.

Use to disable the field import with the given reason.

Source code in nautobot_netbox_importer/generator/fields.py
def disable(reason: str) -> SourceFieldDefinition:
    """Disable the field.

    Use to disable the field import with the given reason.
    """

    def define_disable(field: SourceField) -> None:
        field.disable(reason)

    return define_disable
fallback(value=None, callback=None, nautobot_name='')

Create a fallback field definition.

Use to set a fallback value or callback for the field, if there is an error during the default importer.

Source code in nautobot_netbox_importer/generator/fields.py
def fallback(
    value: Any = None,
    callback: Optional[SourceFieldImporterFallback] = None,
    nautobot_name: FieldName = "",
) -> SourceFieldDefinition:
    """Create a fallback field definition.

    Use to set a fallback value or callback for the field, if there is an error during the default importer.
    """
    if (value is None) == (callback is None):
        raise ValueError("Exactly one of `value` or `callback` must be set.")

    def define_fallback(field: SourceField) -> None:
        original_importer = field.set_importer(nautobot_name=nautobot_name)
        if not original_importer:
            return

        def fallback_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            try:
                original_importer(source, target)
            except Exception as error:
                if callback:
                    callback(field, source, target, error)
                if value:
                    field.set_nautobot_value(target, value)
                    if isinstance(error, InvalidChoiceValueIssue):
                        raise InvalidChoiceValueIssue(field, field.get_source_value(source), value) from error
                    raise SourceFieldImporterIssue(
                        f"Failed to import field: {error} | Fallback value: {value}",
                        field,
                    ) from error
                raise

        field.set_importer(fallback_importer, override=True)

    return define_fallback
force(nautobot_name='')

Mark Nautobot field as forced.

Use to force the field to be saved in Nautobot in the second save attempt after the initial save to override the default value set by Nautobot.

Source code in nautobot_netbox_importer/generator/fields.py
def force(nautobot_name: FieldName = "") -> SourceFieldDefinition:
    """Mark Nautobot field as forced.

    Use to force the field to be saved in Nautobot in the second save attempt after the initial save to override the
    default value set by Nautobot.
    """

    def define_force(field: SourceField) -> None:
        field.set_importer(nautobot_name=nautobot_name)
        field.nautobot.force = True

    return define_force
pass_through(nautobot_name='')

Create a pass-through field definition.

Use to pass-through the value from source to target without changing it by the default importer.

Source code in nautobot_netbox_importer/generator/fields.py
def pass_through(nautobot_name: FieldName = "") -> SourceFieldDefinition:
    """Create a pass-through field definition.

    Use to pass-through the value from source to target without changing it by the default importer.
    """

    def define_passthrough(field: SourceField) -> None:
        def pass_through_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(field.name, None)
            field.set_nautobot_value(target, value)

        field.set_importer(pass_through_importer, nautobot_name)

    return define_passthrough
relation(related_source, nautobot_name='')

Create a relation field definition.

Use when there is a different source content type that should be mapped to Nautobot relation.

Source code in nautobot_netbox_importer/generator/fields.py
def relation(related_source: SourceContentType, nautobot_name: FieldName = "") -> SourceFieldDefinition:
    """Create a relation field definition.

    Use when there is a different source content type that should be mapped to Nautobot relation.
    """

    def define_relation(field: SourceField) -> None:
        field.set_nautobot_field(nautobot_name)
        if field.nautobot.internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
            field.set_m2m_importer(related_source)
        else:
            field.set_relation_importer(related_source)

    return define_relation
role(adapter, source_content_type, nautobot_name='role')

Create a role field definition.

Use, when there is a different source role content type that should be mapped to the Nautobot "extras.Role". It creates a new wrapper for the source_content_type if it does not already exist.

It covers multiple options for how roles can be referenced in the source data
  • by primary key
  • by role name

It also handles the scenario where the same role names are used in different role models, e.g., RackRole with name = "Network" and DeviceRole with name = "Network" to avoid duplicates.

Source code in nautobot_netbox_importer/generator/fields.py
def role(
    adapter: SourceAdapter,
    source_content_type: ContentTypeStr,
    nautobot_name: FieldName = "role",
) -> SourceFieldDefinition:
    """Create a role field definition.

    Use, when there is a different source role content type that should be mapped to the Nautobot "extras.Role".
    It creates a new wrapper for the `source_content_type` if it does not already exist.

    It covers multiple options for how roles can be referenced in the source data:
        - by primary key
        - by role name

    It also handles the scenario where the same role names are used in different role models,
    e.g., RackRole with `name = "Network"` and DeviceRole with `name = "Network"` to avoid duplicates.
    """

    def cache_roles(source: RecordData, importer_pass: ImporterPass) -> PreImportResult:
        if importer_pass == ImporterPass.DEFINE_STRUCTURE:
            name = source.get("name", "").capitalize()
            if not name:
                raise ValueError("Role name is required")
            uid = _ROLE_NAME_TO_UID_CACHE.get(name, None)
            nautobot_uid = role_wrapper.cache_record_uids(source, uid)
            if not uid:
                _ROLE_NAME_TO_UID_CACHE[name] = nautobot_uid

        return PreImportResult.USE_RECORD

    role_wrapper = adapter.configure_model(
        source_content_type,
        nautobot_content_type="extras.role",
        pre_import=cache_roles,
        identifiers=("name",),
        fields={
            # Include color to allow setting the default Nautobot value, import fails without it.
            "color": "color",
        },
    )

    def define_role(field: SourceField) -> None:
        def role_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = field.get_source_value(source)
            if value in EMPTY_VALUES:
                return

            if isinstance(value, (int, UUID)):
                # Role is referenced by primary key
                uid = role_wrapper.get_pk_from_uid(value)
            elif isinstance(value, str):
                # Role is referenced by name
                value = value.capitalize()
                if value in _ROLE_NAME_TO_UID_CACHE:
                    uid = _ROLE_NAME_TO_UID_CACHE[value]
                else:
                    uid = role_wrapper.get_pk_from_identifiers([value])
                    _ROLE_NAME_TO_UID_CACHE[value] = uid
                role_wrapper.import_record({"id": uid, "name": value})
            else:
                raise ValueError(f"Invalid role value {value}")

            field.set_nautobot_value(target, uid)
            field.wrapper.add_reference(role_wrapper, uid)

        field.set_importer(role_importer, nautobot_name)

    return define_role
source_constant(value, nautobot_name='')

Create a source constant field definition.

Use, to pre-fill constant value for the field. Calls default importer after setting the value.

Source code in nautobot_netbox_importer/generator/fields.py
def source_constant(value: Any, nautobot_name: FieldName = "") -> SourceFieldDefinition:
    """Create a source constant field definition.

    Use, to pre-fill constant value for the field. Calls default importer after setting the value.
    """

    def define_source_constant(field: SourceField) -> None:
        original_importer = field.set_importer(nautobot_name=nautobot_name)
        if not original_importer:
            return

        def source_constant_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            source[field.name] = value
            original_importer(source, target)

        field.set_importer(source_constant_importer, override=True)

    return define_source_constant

nautobot

Nautobot DiffSync Importer.

DiffSyncBaseModel

Bases: DiffSyncModel

Base class for all DiffSync models.

Source code in nautobot_netbox_importer/generator/nautobot.py
class DiffSyncBaseModel(DiffSyncModel):
    """Base class for all DiffSync models."""

    _wrapper: NautobotModelWrapper

    @classmethod
    def create(cls, diffsync: DiffSync, ids: dict, attrs: dict) -> Optional["DiffSyncBaseModel"]:
        """Create this model instance, both in Nautobot and in DiffSync."""
        wrapper = cls._wrapper

        instance = None
        try:
            instance = wrapper.model(**wrapper.constructor_kwargs, **ids)
            result = super().create(diffsync, ids, attrs)
        # pylint: disable=broad-exception-caught
        except Exception as error:
            wrapper.add_issue(
                "CreateFailed",
                uid=ids.get(wrapper.pk_field.name, ""),
                nautobot_instance=instance,
                data=attrs,
                error=error,
            )
            return None

        if wrapper.save_nautobot_instance(instance, attrs):
            wrapper.stats.created += 1
            return result

        return None

    def update(self, attrs: dict) -> Optional["DiffSyncBaseModel"]:
        """Update this model instance, both in Nautobot and in DiffSync."""
        wrapper = self._wrapper
        uid = getattr(self, wrapper.pk_field.name, None)
        if not uid:
            raise NotImplementedError("Cannot update model without pk")

        try:
            super().update(attrs)
        # pylint: disable=broad-exception-caught
        except Exception as error:
            wrapper.add_issue(
                "UpdateFailed",
                uid=uid,
                diffsync_instance=self,
                data=attrs,
                error=error,
            )
            return None

        model = wrapper.model
        filter_kwargs = {wrapper.pk_field.name: uid}
        instance = model.objects.get(**filter_kwargs)
        if wrapper.save_nautobot_instance(instance, attrs):
            wrapper.stats.updated += 1
            return self

        return None
create(diffsync, ids, attrs) classmethod

Create this model instance, both in Nautobot and in DiffSync.

Source code in nautobot_netbox_importer/generator/nautobot.py
@classmethod
def create(cls, diffsync: DiffSync, ids: dict, attrs: dict) -> Optional["DiffSyncBaseModel"]:
    """Create this model instance, both in Nautobot and in DiffSync."""
    wrapper = cls._wrapper

    instance = None
    try:
        instance = wrapper.model(**wrapper.constructor_kwargs, **ids)
        result = super().create(diffsync, ids, attrs)
    # pylint: disable=broad-exception-caught
    except Exception as error:
        wrapper.add_issue(
            "CreateFailed",
            uid=ids.get(wrapper.pk_field.name, ""),
            nautobot_instance=instance,
            data=attrs,
            error=error,
        )
        return None

    if wrapper.save_nautobot_instance(instance, attrs):
        wrapper.stats.created += 1
        return result

    return None
update(attrs)

Update this model instance, both in Nautobot and in DiffSync.

Source code in nautobot_netbox_importer/generator/nautobot.py
def update(self, attrs: dict) -> Optional["DiffSyncBaseModel"]:
    """Update this model instance, both in Nautobot and in DiffSync."""
    wrapper = self._wrapper
    uid = getattr(self, wrapper.pk_field.name, None)
    if not uid:
        raise NotImplementedError("Cannot update model without pk")

    try:
        super().update(attrs)
    # pylint: disable=broad-exception-caught
    except Exception as error:
        wrapper.add_issue(
            "UpdateFailed",
            uid=uid,
            diffsync_instance=self,
            data=attrs,
            error=error,
        )
        return None

    model = wrapper.model
    filter_kwargs = {wrapper.pk_field.name: uid}
    instance = model.objects.get(**filter_kwargs)
    if wrapper.save_nautobot_instance(instance, attrs):
        wrapper.stats.updated += 1
        return self

    return None
NautobotAdapter

Bases: BaseAdapter

Nautobot DiffSync Adapter.

Source code in nautobot_netbox_importer/generator/nautobot.py
class NautobotAdapter(BaseAdapter):
    """Nautobot DiffSync Adapter."""

    def __init__(self, *args, **kwargs):
        """Initialize the adapter."""
        super().__init__("Nautobot", *args, **kwargs)
        self.wrappers: Dict[ContentTypeStr, NautobotModelWrapper] = {}
        self.trace_issues = False

    def get_or_create_wrapper(self, content_type: ContentTypeStr) -> "NautobotModelWrapper":
        """Get or create a Nautobot model wrapper."""
        if content_type in self.wrappers:
            return self.wrappers[content_type]

        return NautobotModelWrapper(self, content_type)
__init__(*args, **kwargs)

Initialize the adapter.

Source code in nautobot_netbox_importer/generator/nautobot.py
def __init__(self, *args, **kwargs):
    """Initialize the adapter."""
    super().__init__("Nautobot", *args, **kwargs)
    self.wrappers: Dict[ContentTypeStr, NautobotModelWrapper] = {}
    self.trace_issues = False
get_or_create_wrapper(content_type)

Get or create a Nautobot model wrapper.

Source code in nautobot_netbox_importer/generator/nautobot.py
def get_or_create_wrapper(self, content_type: ContentTypeStr) -> "NautobotModelWrapper":
    """Get or create a Nautobot model wrapper."""
    if content_type in self.wrappers:
        return self.wrappers[content_type]

    return NautobotModelWrapper(self, content_type)
NautobotField

Wrapper for a Nautobot field.

Source code in nautobot_netbox_importer/generator/nautobot.py
class NautobotField:
    """Wrapper for a Nautobot field."""

    def __init__(self, name: FieldName, internal_type: InternalFieldType, field: Optional[DjangoField] = None):
        """Initialize the wrapper."""
        self.name = name
        self.internal_type = internal_type
        self.field = field
        self.required = not (getattr(field, "null", False) or getattr(field, "blank", False)) if field else False

        # Forced fields needs to be saved in a separate step after the initial save.
        self.force = self.name == "created"

    def __str__(self) -> str:
        """Return a string representation of the wrapper."""
        return f"{self.__class__.__name__}<{self.name} {self.internal_type}>"

    @property
    def related_model(self) -> NautobotBaseModelType:
        """Get the related model for a reference field."""
        if not isinstance(self.field, DjangoField):
            raise NotImplementedError(f"Unsupported relation importer {self}")

        return getattr(self.field, "related_model")

    @property
    def related_meta(self) -> DjangoModelMeta:
        """Get the Nautobot model meta."""
        return self.related_model._meta  # type: ignore

    @property
    def is_reference(self) -> bool:
        """Check if the field is a reference."""
        return self.internal_type in _REFERENCE_TYPES

    @property
    def is_integer(self) -> bool:
        """Check if the field is an integer."""
        return self.internal_type in _INTEGER_TYPES

    @property
    def is_auto_increment(self) -> bool:
        """Check if the field is an integer."""
        return self.internal_type in _AUTO_INCREMENT_TYPES

    @property
    def is_content_type(self) -> bool:
        """Check if the field is a content type."""
        if not self.is_reference:
            return False

        return self.related_model == ContentType

    @property
    def can_import(self) -> bool:
        """Determine if this field can be imported."""
        return self.internal_type not in _DONT_IMPORT_TYPES
can_import: bool property

Determine if this field can be imported.

is_auto_increment: bool property

Check if the field is an integer.

is_content_type: bool property

Check if the field is a content type.

is_integer: bool property

Check if the field is an integer.

is_reference: bool property

Check if the field is a reference.

related_meta: DjangoModelMeta property

Get the Nautobot model meta.

related_model: NautobotBaseModelType property

Get the related model for a reference field.

__init__(name, internal_type, field=None)

Initialize the wrapper.

Source code in nautobot_netbox_importer/generator/nautobot.py
def __init__(self, name: FieldName, internal_type: InternalFieldType, field: Optional[DjangoField] = None):
    """Initialize the wrapper."""
    self.name = name
    self.internal_type = internal_type
    self.field = field
    self.required = not (getattr(field, "null", False) or getattr(field, "blank", False)) if field else False

    # Forced fields needs to be saved in a separate step after the initial save.
    self.force = self.name == "created"
__str__()

Return a string representation of the wrapper.

Source code in nautobot_netbox_importer/generator/nautobot.py
def __str__(self) -> str:
    """Return a string representation of the wrapper."""
    return f"{self.__class__.__name__}<{self.name} {self.internal_type}>"
NautobotModelWrapper

Wrapper for a Nautobot model.

Source code in nautobot_netbox_importer/generator/nautobot.py
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
class NautobotModelWrapper:
    """Wrapper for a Nautobot model."""

    def __init__(self, adapter: NautobotAdapter, content_type: ContentTypeStr):
        """Initialize the wrapper."""
        self._diffsync_class: Optional[Type[DiffSyncBaseModel]] = None
        self._issues: List[ImporterIssue] = []
        self._content_type_instance = None
        self.flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST

        self.adapter = adapter
        adapter.wrappers[content_type] = self
        self.content_type = content_type
        try:
            self._model = get_model_from_name(content_type)
            self.disabled = False
        except TypeError:
            self._model = None
            self.disabled = True

        self.fields: NautobotFields = {}

        self.last_id = 0

        if self.disabled:
            logger.info("Skipping unknown model %s", content_type)
            self._pk_field = None
        else:
            self._pk_field = self.add_field(self.model_meta.pk.name)  # type: ignore
            if not self._pk_field:
                raise ValueError(f"Missing pk field for {self.content_type}")

            if self._pk_field.is_auto_increment:
                self.last_id = (
                    self.model.objects.aggregate(Max(self._pk_field.name))[f"{self._pk_field.name}__max"] or 0
                )

            for field_name in AUTO_ADD_FIELDS:
                if hasattr(self.model, field_name):
                    self.add_field(field_name)

        self.constructor_kwargs: Dict[FieldName, Any] = {}
        self.stats = NautobotModelStats()

        logger.debug("Created %s", self)

    def __str__(self) -> str:
        """Return a string representation of the wrapper."""
        return f"{self.__class__.__name__}<{self.content_type}>"

    @property
    def pk_field(self) -> NautobotField:
        """Get the pk field."""
        if not self._pk_field:
            raise ValueError(f"Missing pk field for {self.content_type}")
        return self._pk_field

    @property
    def model(self) -> NautobotBaseModelType:
        """Get the Nautobot model."""
        if self._model:
            return self._model
        raise NautobotModelNotFound(self.content_type)

    @property
    def model_meta(self) -> DjangoModelMeta:
        """Get the Nautobot model meta."""
        return self.model._meta  # type: ignore

    @property
    def diffsync_class(self) -> Type["DiffSyncBaseModel"]:
        """Get `DiffSyncModel` class for this wrapper."""
        if self._diffsync_class:
            return self._diffsync_class

        if self.disabled:
            raise RuntimeError("Cannot create importer for disabled model")

        annotations = {}
        attributes = []
        identifiers = []

        class_definition = {
            "__annotations__": annotations,
            "_attributes": attributes,
            "_identifiers": identifiers,
            "_modelname": self.content_type.replace(".", "_"),
            "_wrapper": self,
        }

        annotations[self.pk_field.name] = INTERNAL_TYPE_TO_ANNOTATION[self.pk_field.internal_type]
        identifiers.append(self.pk_field.name)
        class_definition[self.pk_field.name] = PydanticField()

        for field in self.fields.values():
            if field.name in identifiers or not field.can_import:
                continue

            if field.is_reference:
                related_type = StrToInternalFieldType[field.related_meta.pk.get_internal_type()]  # type: ignore
                annotation = INTERNAL_TYPE_TO_ANNOTATION[related_type]
                if field.internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
                    annotation = Set[annotation]
            else:
                annotation = INTERNAL_TYPE_TO_ANNOTATION[field.internal_type]

            attributes.append(field.name)
            if not field.required:
                annotation = Optional[annotation]
            annotations[field.name] = annotation
            class_definition[field.name] = PydanticField(default=None)

        try:
            result = type(class_definition["_modelname"], (DiffSyncBaseModel,), class_definition)
            self._diffsync_class = result
        except Exception:
            logger.error("Failed to create DiffSync Model %s", class_definition, exc_info=True)
            raise

        logger.debug("Created DiffSync Model %s", class_definition)

        return result

    def get_summary(self) -> NautobotModelSummary:
        """Get the summary."""
        issues = sorted(self.get_importer_issues())
        if issues:
            self.stats.issues = len(issues)

        return NautobotModelSummary(
            content_type=self.content_type,
            content_type_id=None if self.disabled else self.content_type_instance.pk,
            stats=self.stats,
            issues=issues,
            flags=str(self.flags),
            disabled=self.disabled,
        )

    # pylint: disable=too-many-arguments
    def add_issue(  # noqa: PLR0913
        self,
        issue_type="",
        message="",
        uid: Uid = "",
        data: Optional[Mapping] = None,
        diffsync_instance: Optional[DiffSyncModel] = None,
        nautobot_instance: Optional[NautobotBaseModel] = None,
        error: Optional[Exception] = None,
        nautobot_name="",
    ) -> ImporterIssue:
        """Add a new importer issue.

        This function register an issue and returns the issue object. All input arguments are optional, internal logic
        tries to fill in as much information as possible based on the arguments provided.

        It can be called in any import stage.

        Args:
            issue_type (Optional[str]): The type of the issue, e.g. "SaveFailed". Can be determined from `error.__class__`.
            message (Optional[str]): A message to be included in the issue. Can be determined from `error`.
            uid (Optional[Uid]): The UID of the instance that caused the issue. Can be determined from instances.
            data (Optional[Mapping]): The data that caused the issue.
            diffsync_instance (Optional[DiffSyncModel]): The DiffSync instance that caused the issue.
            nautobot_instance (Optional[NautobotBaseModel]): The Nautobot instance that caused the issue.
            error (Optional[Exception]): The error that caused the issue.
            nautobot_name (Optional[str]): The name of the Nautobot instance that caused the issue.
                This is determined after the import, before creating summaries.

        Examples can be found by looking at calls to this function in the code.
        """
        issue = self._create_issue(
            issue_type, message, uid, data, diffsync_instance, nautobot_instance, error, nautobot_name
        )
        self._issues.append(issue)
        return issue

    # pylint: disable=too-many-arguments,too-many-branches
    def _create_issue(  # noqa: PLR0912, PLR0913
        self,
        issue_type="",
        message="",
        uid: Uid = "",
        data: Optional[Mapping] = None,
        diffsync_instance: Optional[DiffSyncModel] = None,
        nautobot_instance: Optional[NautobotBaseModel] = None,
        error: Optional[Exception] = None,
        nautobot_name="",
    ) -> ImporterIssue:
        """Create an issue."""
        if not issue_type:
            if error:
                issue_type = error.__class__.__name__
            else:
                issue_type = "Unknown"

        if diffsync_instance:
            if not uid:
                uid = getattr(diffsync_instance, self.pk_field.name)
            if not data:
                data = diffsync_instance.__dict__

        if nautobot_instance:
            if not uid:
                uid = getattr(nautobot_instance, self.pk_field.name)
            if not data:
                data = nautobot_instance.__dict__
            if not nautobot_name:
                try:
                    nautobot_name = str(nautobot_instance)
                # pylint: disable=broad-exception-caught
                except Exception:  # noqa: S110
                    # Can happen for non-complete Nautobot instances. Pass silently, just don't set the name.
                    pass

        # Convert data to `dict[str, str]`
        data_dict = {
            key: str(value) for key, value in (data or {}).items() if isinstance(key, str) and not key.startswith("_")
        }

        if not uid:
            uid = data_dict.get(self.pk_field.name, "")

        def get_message():
            if message:
                yield message
            if error:
                yield str(error)

        issue = ImporterIssue(
            uid=str(uid),
            name=nautobot_name,
            issue_type=issue_type,
            message=" ".join(get_message()),
            data=data_dict,
        )

        logger.warning(str(issue))
        if error and self.adapter.trace_issues:
            logger.error("Issue traceback", exc_info=error)

        return issue

    def find_or_create(self, identifiers_kwargs: dict) -> Optional[DiffSyncModel]:
        """Find a DiffSync instance based on filter kwargs or create a new instance from Nautobot if possible."""
        result = self.adapter.get_or_none(self.diffsync_class, identifiers_kwargs)
        if result:
            return result

        try:
            nautobot_instance = self.model.objects.get(**identifiers_kwargs)
        except self.model.DoesNotExist:  # type: ignore
            return None

        diffsync_class = self.diffsync_class

        uid = getattr(nautobot_instance, self.pk_field.name)
        kwargs = {self.pk_field.name: uid}
        if identifiers_kwargs != kwargs:
            result = self.adapter.get_or_none(diffsync_class, kwargs)
            if result:
                return result

        result = diffsync_class(**kwargs, diffsync=self.adapter)  # type: ignore
        result.model_flags = self.flags
        self._nautobot_to_diffsync(nautobot_instance, result)
        self.adapter.add(result)

        return result

    def get_importer_issues(self) -> List[ImporterIssue]:
        """Get importer issues for this model.

        This will also run `clean` on all instances that failed `clean()` after saving.
        """
        result = []

        for issue in self._issues:
            if not issue.uid or (issue.name and issue.issue_type != "CleanFailed"):
                # Just copy an issue, can't, or doesn't need to, be cleaned or extended.
                result.append(issue)
                continue

            try:
                nautobot_instance = self.model.objects.get(id=issue.uid)
            except self.model.DoesNotExist as error:  # type: ignore
                if issue.issue_type == "CleanFailed":
                    # This can happen with Tree models, not sure why. Ignore for now, just add the importer issue.
                    result.append(
                        self._create_issue(
                            uid=issue.uid,
                            error=error,
                            message="Instance was not found, even it was saved.",
                        )
                    )
                result.append(issue)
                continue

            if issue.issue_type == "CleanFailed":
                # Re-run clean on the instance, add the issue if it fails.
                try:
                    nautobot_instance.clean()
                # pylint: disable=broad-exception-caught
                except Exception as error:
                    result.append(
                        self._create_issue(
                            uid=issue.uid,
                            data=issue.data,
                            nautobot_instance=nautobot_instance,
                            error=error,
                        )
                    )
                continue

            # Extend the issue with the Nautobot instance.
            result.append(
                self._create_issue(
                    issue_type=issue.issue_type,
                    message=issue.message,
                    uid=issue.uid,
                    data=issue.data,
                    nautobot_instance=nautobot_instance,
                )
            )

        self._issues = []

        return result

    def add_field(self, field_name: FieldName) -> NautobotField:
        """Add a field to the model."""
        if self._diffsync_class:
            raise RuntimeError("Cannot add fields after the DiffSync Model has been created")

        nautobot_field, internal_type = get_nautobot_field_and_type(self.model, field_name)

        if (
            internal_type in _REFERENCE_TYPES
            and internal_type != InternalFieldType.MANY_TO_MANY_FIELD
            and not field_name.endswith("_id")
        ):
            # Reference fields are converted to id fields
            field_name = f"{field_name}_id"

        if field_name in self.fields:
            field = self.fields[field_name]
            if field.internal_type != internal_type:
                raise ValueError(f"Field {field_name} already exists with different type {self.fields[field_name]}")
        else:
            logger.debug("Adding nautobot field %s %s %s", self.content_type, field_name, internal_type)
            field = NautobotField(field_name, internal_type, nautobot_field)
            self.fields[field_name] = field

        return field

    @property
    def content_type_instance(self) -> ContentType:
        """Get the Nautobot content type instance for a given content type."""
        if not self._content_type_instance:
            self._content_type_instance = ContentType.objects.get_for_model(self.model)
        return self._content_type_instance

    def set_instance_defaults(self, **defaults: Any) -> None:
        """Set default values for a Nautobot instance constructor."""
        self.constructor_kwargs = defaults

    def _nautobot_to_diffsync(self, source: NautobotBaseModel, target: "DiffSyncBaseModel") -> None:
        """Copy data from Nautobot instance to DiffSync Model."""

        def set_value(field_name, internal_type) -> None:
            value = getattr(source, field_name, None)
            if value in EMPTY_VALUES:
                return

            if internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
                values = value.all()  # type: ignore
                if values:
                    setattr(target, field_name, set(item.pk for item in values))
            elif internal_type == InternalFieldType.PROPERTY:
                setattr(target, field_name, str(value))
            elif internal_type == InternalFieldType.DATE_TIME_FIELD:
                setattr(target, field_name, normalize_datetime(value))
            else:
                setattr(target, field_name, value)

        for field in self.fields.values():
            if field.can_import:
                set_value(field.name, field.internal_type)

    def save_nautobot_instance(self, instance: NautobotBaseModel, values: RecordData) -> bool:
        """Save a Nautobot instance."""

        def set_custom_field_data(value: Optional[Mapping]):
            custom_field_data = getattr(instance, "custom_field_data", None)
            if custom_field_data is None:
                raise TypeError("Missing custom_field_data")
            custom_field_data.clear()
            if value:
                custom_field_data.update(value)

        def set_empty(field, field_name: FieldName):
            if field.blank and not field.null:
                setattr(instance, field_name, "")
            else:
                setattr(instance, field_name, None)

        @atomic
        def save():
            m2m_fields = set()
            force_fields = {}

            for field_name, value in values.items():
                field_wrapper = self.fields[field_name]
                if not field_wrapper.can_import:
                    continue

                if field_wrapper.internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
                    m2m_fields.add(field_name)
                elif field_wrapper.internal_type == InternalFieldType.CUSTOM_FIELD_DATA:
                    set_custom_field_data(value)
                elif value in EMPTY_VALUES:
                    set_empty(field_wrapper.field, field_name)
                elif field_wrapper.force:
                    force_fields[field_name] = value
                else:
                    setattr(instance, field_name, value)

            instance.save()

            if force_fields:
                # These fields has to be set after the initial save to override any default values.
                for field_name, value in force_fields.items():
                    setattr(instance, field_name, value)
                instance.save()

            for field_name in m2m_fields:
                field = getattr(instance, field_name)
                value = values[field_name]
                if value:
                    field.set(value)
                else:
                    field.clear()

        try:
            save()
        # pylint: disable=broad-exception-caught
        except Exception as error:
            self.stats.save_failed += 1
            self.add_issue(
                "SaveFailed",
                nautobot_instance=instance,
                data=values,
                error=error,
            )
            return False

        try:
            instance.clean()
        # pylint: disable=broad-exception-caught
        except Exception as error:
            # `clean()` is called again by `get_importer_issues()` after importing all data
            self.add_issue(
                "CleanFailed",
                nautobot_instance=instance,
                data=values,
                error=error,
            )

        return True
content_type_instance: ContentType property

Get the Nautobot content type instance for a given content type.

diffsync_class: Type[DiffSyncBaseModel] property

Get DiffSyncModel class for this wrapper.

model: NautobotBaseModelType property

Get the Nautobot model.

model_meta: DjangoModelMeta property

Get the Nautobot model meta.

pk_field: NautobotField property

Get the pk field.

__init__(adapter, content_type)

Initialize the wrapper.

Source code in nautobot_netbox_importer/generator/nautobot.py
def __init__(self, adapter: NautobotAdapter, content_type: ContentTypeStr):
    """Initialize the wrapper."""
    self._diffsync_class: Optional[Type[DiffSyncBaseModel]] = None
    self._issues: List[ImporterIssue] = []
    self._content_type_instance = None
    self.flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST

    self.adapter = adapter
    adapter.wrappers[content_type] = self
    self.content_type = content_type
    try:
        self._model = get_model_from_name(content_type)
        self.disabled = False
    except TypeError:
        self._model = None
        self.disabled = True

    self.fields: NautobotFields = {}

    self.last_id = 0

    if self.disabled:
        logger.info("Skipping unknown model %s", content_type)
        self._pk_field = None
    else:
        self._pk_field = self.add_field(self.model_meta.pk.name)  # type: ignore
        if not self._pk_field:
            raise ValueError(f"Missing pk field for {self.content_type}")

        if self._pk_field.is_auto_increment:
            self.last_id = (
                self.model.objects.aggregate(Max(self._pk_field.name))[f"{self._pk_field.name}__max"] or 0
            )

        for field_name in AUTO_ADD_FIELDS:
            if hasattr(self.model, field_name):
                self.add_field(field_name)

    self.constructor_kwargs: Dict[FieldName, Any] = {}
    self.stats = NautobotModelStats()

    logger.debug("Created %s", self)
__str__()

Return a string representation of the wrapper.

Source code in nautobot_netbox_importer/generator/nautobot.py
def __str__(self) -> str:
    """Return a string representation of the wrapper."""
    return f"{self.__class__.__name__}<{self.content_type}>"
add_field(field_name)

Add a field to the model.

Source code in nautobot_netbox_importer/generator/nautobot.py
def add_field(self, field_name: FieldName) -> NautobotField:
    """Add a field to the model."""
    if self._diffsync_class:
        raise RuntimeError("Cannot add fields after the DiffSync Model has been created")

    nautobot_field, internal_type = get_nautobot_field_and_type(self.model, field_name)

    if (
        internal_type in _REFERENCE_TYPES
        and internal_type != InternalFieldType.MANY_TO_MANY_FIELD
        and not field_name.endswith("_id")
    ):
        # Reference fields are converted to id fields
        field_name = f"{field_name}_id"

    if field_name in self.fields:
        field = self.fields[field_name]
        if field.internal_type != internal_type:
            raise ValueError(f"Field {field_name} already exists with different type {self.fields[field_name]}")
    else:
        logger.debug("Adding nautobot field %s %s %s", self.content_type, field_name, internal_type)
        field = NautobotField(field_name, internal_type, nautobot_field)
        self.fields[field_name] = field

    return field
add_issue(issue_type='', message='', uid='', data=None, diffsync_instance=None, nautobot_instance=None, error=None, nautobot_name='')

Add a new importer issue.

This function register an issue and returns the issue object. All input arguments are optional, internal logic tries to fill in as much information as possible based on the arguments provided.

It can be called in any import stage.

Parameters:

Name Type Description Default
issue_type Optional[str]

The type of the issue, e.g. "SaveFailed". Can be determined from error.__class__.

''
message Optional[str]

A message to be included in the issue. Can be determined from error.

''
uid Optional[Uid]

The UID of the instance that caused the issue. Can be determined from instances.

''
data Optional[Mapping]

The data that caused the issue.

None
diffsync_instance Optional[DiffSyncModel]

The DiffSync instance that caused the issue.

None
nautobot_instance Optional[NautobotBaseModel]

The Nautobot instance that caused the issue.

None
error Optional[Exception]

The error that caused the issue.

None
nautobot_name Optional[str]

The name of the Nautobot instance that caused the issue. This is determined after the import, before creating summaries.

''

Examples can be found by looking at calls to this function in the code.

Source code in nautobot_netbox_importer/generator/nautobot.py
def add_issue(  # noqa: PLR0913
    self,
    issue_type="",
    message="",
    uid: Uid = "",
    data: Optional[Mapping] = None,
    diffsync_instance: Optional[DiffSyncModel] = None,
    nautobot_instance: Optional[NautobotBaseModel] = None,
    error: Optional[Exception] = None,
    nautobot_name="",
) -> ImporterIssue:
    """Add a new importer issue.

    This function register an issue and returns the issue object. All input arguments are optional, internal logic
    tries to fill in as much information as possible based on the arguments provided.

    It can be called in any import stage.

    Args:
        issue_type (Optional[str]): The type of the issue, e.g. "SaveFailed". Can be determined from `error.__class__`.
        message (Optional[str]): A message to be included in the issue. Can be determined from `error`.
        uid (Optional[Uid]): The UID of the instance that caused the issue. Can be determined from instances.
        data (Optional[Mapping]): The data that caused the issue.
        diffsync_instance (Optional[DiffSyncModel]): The DiffSync instance that caused the issue.
        nautobot_instance (Optional[NautobotBaseModel]): The Nautobot instance that caused the issue.
        error (Optional[Exception]): The error that caused the issue.
        nautobot_name (Optional[str]): The name of the Nautobot instance that caused the issue.
            This is determined after the import, before creating summaries.

    Examples can be found by looking at calls to this function in the code.
    """
    issue = self._create_issue(
        issue_type, message, uid, data, diffsync_instance, nautobot_instance, error, nautobot_name
    )
    self._issues.append(issue)
    return issue
find_or_create(identifiers_kwargs)

Find a DiffSync instance based on filter kwargs or create a new instance from Nautobot if possible.

Source code in nautobot_netbox_importer/generator/nautobot.py
def find_or_create(self, identifiers_kwargs: dict) -> Optional[DiffSyncModel]:
    """Find a DiffSync instance based on filter kwargs or create a new instance from Nautobot if possible."""
    result = self.adapter.get_or_none(self.diffsync_class, identifiers_kwargs)
    if result:
        return result

    try:
        nautobot_instance = self.model.objects.get(**identifiers_kwargs)
    except self.model.DoesNotExist:  # type: ignore
        return None

    diffsync_class = self.diffsync_class

    uid = getattr(nautobot_instance, self.pk_field.name)
    kwargs = {self.pk_field.name: uid}
    if identifiers_kwargs != kwargs:
        result = self.adapter.get_or_none(diffsync_class, kwargs)
        if result:
            return result

    result = diffsync_class(**kwargs, diffsync=self.adapter)  # type: ignore
    result.model_flags = self.flags
    self._nautobot_to_diffsync(nautobot_instance, result)
    self.adapter.add(result)

    return result
get_importer_issues()

Get importer issues for this model.

This will also run clean on all instances that failed clean() after saving.

Source code in nautobot_netbox_importer/generator/nautobot.py
def get_importer_issues(self) -> List[ImporterIssue]:
    """Get importer issues for this model.

    This will also run `clean` on all instances that failed `clean()` after saving.
    """
    result = []

    for issue in self._issues:
        if not issue.uid or (issue.name and issue.issue_type != "CleanFailed"):
            # Just copy an issue, can't, or doesn't need to, be cleaned or extended.
            result.append(issue)
            continue

        try:
            nautobot_instance = self.model.objects.get(id=issue.uid)
        except self.model.DoesNotExist as error:  # type: ignore
            if issue.issue_type == "CleanFailed":
                # This can happen with Tree models, not sure why. Ignore for now, just add the importer issue.
                result.append(
                    self._create_issue(
                        uid=issue.uid,
                        error=error,
                        message="Instance was not found, even it was saved.",
                    )
                )
            result.append(issue)
            continue

        if issue.issue_type == "CleanFailed":
            # Re-run clean on the instance, add the issue if it fails.
            try:
                nautobot_instance.clean()
            # pylint: disable=broad-exception-caught
            except Exception as error:
                result.append(
                    self._create_issue(
                        uid=issue.uid,
                        data=issue.data,
                        nautobot_instance=nautobot_instance,
                        error=error,
                    )
                )
            continue

        # Extend the issue with the Nautobot instance.
        result.append(
            self._create_issue(
                issue_type=issue.issue_type,
                message=issue.message,
                uid=issue.uid,
                data=issue.data,
                nautobot_instance=nautobot_instance,
            )
        )

    self._issues = []

    return result
get_summary()

Get the summary.

Source code in nautobot_netbox_importer/generator/nautobot.py
def get_summary(self) -> NautobotModelSummary:
    """Get the summary."""
    issues = sorted(self.get_importer_issues())
    if issues:
        self.stats.issues = len(issues)

    return NautobotModelSummary(
        content_type=self.content_type,
        content_type_id=None if self.disabled else self.content_type_instance.pk,
        stats=self.stats,
        issues=issues,
        flags=str(self.flags),
        disabled=self.disabled,
    )
save_nautobot_instance(instance, values)

Save a Nautobot instance.

Source code in nautobot_netbox_importer/generator/nautobot.py
def save_nautobot_instance(self, instance: NautobotBaseModel, values: RecordData) -> bool:
    """Save a Nautobot instance."""

    def set_custom_field_data(value: Optional[Mapping]):
        custom_field_data = getattr(instance, "custom_field_data", None)
        if custom_field_data is None:
            raise TypeError("Missing custom_field_data")
        custom_field_data.clear()
        if value:
            custom_field_data.update(value)

    def set_empty(field, field_name: FieldName):
        if field.blank and not field.null:
            setattr(instance, field_name, "")
        else:
            setattr(instance, field_name, None)

    @atomic
    def save():
        m2m_fields = set()
        force_fields = {}

        for field_name, value in values.items():
            field_wrapper = self.fields[field_name]
            if not field_wrapper.can_import:
                continue

            if field_wrapper.internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
                m2m_fields.add(field_name)
            elif field_wrapper.internal_type == InternalFieldType.CUSTOM_FIELD_DATA:
                set_custom_field_data(value)
            elif value in EMPTY_VALUES:
                set_empty(field_wrapper.field, field_name)
            elif field_wrapper.force:
                force_fields[field_name] = value
            else:
                setattr(instance, field_name, value)

        instance.save()

        if force_fields:
            # These fields has to be set after the initial save to override any default values.
            for field_name, value in force_fields.items():
                setattr(instance, field_name, value)
            instance.save()

        for field_name in m2m_fields:
            field = getattr(instance, field_name)
            value = values[field_name]
            if value:
                field.set(value)
            else:
                field.clear()

    try:
        save()
    # pylint: disable=broad-exception-caught
    except Exception as error:
        self.stats.save_failed += 1
        self.add_issue(
            "SaveFailed",
            nautobot_instance=instance,
            data=values,
            error=error,
        )
        return False

    try:
        instance.clean()
    # pylint: disable=broad-exception-caught
    except Exception as error:
        # `clean()` is called again by `get_importer_issues()` after importing all data
        self.add_issue(
            "CleanFailed",
            nautobot_instance=instance,
            data=values,
            error=error,
        )

    return True
set_instance_defaults(**defaults)

Set default values for a Nautobot instance constructor.

Source code in nautobot_netbox_importer/generator/nautobot.py
def set_instance_defaults(self, **defaults: Any) -> None:
    """Set default values for a Nautobot instance constructor."""
    self.constructor_kwargs = defaults

source

Generic DiffSync Source Generator.

ImporterPass

Bases: Enum

Importer Pass.

Source code in nautobot_netbox_importer/generator/source.py
class ImporterPass(Enum):
    """Importer Pass."""

    DEFINE_STRUCTURE = 1
    IMPORT_DATA = 2
InvalidChoiceValueIssue

Bases: SourceFieldImporterIssue

Raised when an invalid choice value is encountered.

Source code in nautobot_netbox_importer/generator/source.py
class InvalidChoiceValueIssue(SourceFieldImporterIssue):
    """Raised when an invalid choice value is encountered."""

    def __init__(self, field: "SourceField", value: Any, replacement: Any = NOTHING):
        """Initialize the exception."""
        message = f"Invalid choice value: `{value}`"
        if replacement is not NOTHING:
            message += f", replaced with `{replacement}`"
        super().__init__(message, field)
__init__(field, value, replacement=NOTHING)

Initialize the exception.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(self, field: "SourceField", value: Any, replacement: Any = NOTHING):
    """Initialize the exception."""
    message = f"Invalid choice value: `{value}`"
    if replacement is not NOTHING:
        message += f", replaced with `{replacement}`"
    super().__init__(message, field)
PreImportResult

Bases: Enum

Pre Import Response.

Source code in nautobot_netbox_importer/generator/source.py
class PreImportResult(Enum):
    """Pre Import Response."""

    SKIP_RECORD = False
    USE_RECORD = True
SourceAdapter

Bases: BaseAdapter

Source DiffSync Adapter.

Source code in nautobot_netbox_importer/generator/source.py
class SourceAdapter(BaseAdapter):
    """Source DiffSync Adapter."""

    def __init__(
        self,
        *args,
        get_source_data: SourceDataGenerator,
        trace_issues: bool = False,
        nautobot: Optional[NautobotAdapter] = None,
        logger=None,
        **kwargs,
    ):
        """Initialize the SourceAdapter."""
        super().__init__(*args, **kwargs)

        self.get_source_data = get_source_data
        self.wrappers: OrderedDict[ContentTypeStr, SourceModelWrapper] = OrderedDict()
        self.nautobot = nautobot or NautobotAdapter()
        self.nautobot.trace_issues = trace_issues
        self.content_type_ids_mapping: Dict[int, SourceModelWrapper] = {}
        self.logger = logger or default_logger
        self.summary = ImportSummary()

        # From Nautobot to Source content type mapping
        # When multiple source content types are mapped to the single nautobot content type, mapping is set to `None`
        self._content_types_back_mapping: Dict[ContentTypeStr, Optional[ContentTypeStr]] = {}

    # pylint: disable=too-many-arguments,too-many-branches,too-many-locals
    def configure_model(
        self,
        content_type: ContentTypeStr,
        nautobot_content_type: ContentTypeStr = "",
        extend_content_type: ContentTypeStr = "",
        identifiers: Optional[Iterable[FieldName]] = None,
        fields: Optional[Mapping[FieldName, SourceFieldDefinition]] = None,
        default_reference: Optional[RecordData] = None,
        flags: Optional[DiffSyncModelFlags] = None,
        nautobot_flags: Optional[DiffSyncModelFlags] = None,
        pre_import: Optional[PreImport] = None,
        disable_related_reference: Optional[bool] = None,
        forward_references: Optional[ForwardReferences] = None,
    ) -> "SourceModelWrapper":
        """Create if not exist and configure a wrapper for a given source content type.

        Create Nautobot content type wrapper as well.
        """
        content_type = content_type.lower()
        nautobot_content_type = nautobot_content_type.lower()
        extend_content_type = extend_content_type.lower()

        if extend_content_type:
            if nautobot_content_type:
                raise ValueError(f"Can't specify both nautobot_content_type and extend_content_type {content_type}")
            extends_wrapper = self.wrappers[extend_content_type]
            nautobot_content_type = extends_wrapper.nautobot.content_type
        else:
            extends_wrapper = None

        if content_type in self.wrappers:
            wrapper = self.wrappers[content_type]
            if nautobot_content_type and wrapper.nautobot.content_type != nautobot_content_type:
                raise ValueError(
                    f"Content type {content_type} already mapped to {wrapper.nautobot.content_type} "
                    f"can't map to {nautobot_content_type}"
                )
        else:
            nautobot_wrapper = self.nautobot.get_or_create_wrapper(nautobot_content_type or content_type)
            wrapper = SourceModelWrapper(self, content_type, nautobot_wrapper)
            if not extends_wrapper:
                if nautobot_wrapper.content_type in self._content_types_back_mapping:
                    if self._content_types_back_mapping[nautobot_wrapper.content_type] != content_type:
                        self._content_types_back_mapping[nautobot_wrapper.content_type] = None
                else:
                    self._content_types_back_mapping[nautobot_wrapper.content_type] = content_type

        if extends_wrapper:
            wrapper.extends_wrapper = extends_wrapper

        if identifiers:
            wrapper.set_identifiers(identifiers)
        for field_name, definition in (fields or {}).items():
            wrapper.add_field(field_name, SourceFieldSource.CUSTOM).set_definition(definition)
        if default_reference:
            wrapper.set_default_reference(default_reference)
        if flags is not None:
            wrapper.flags = flags
        if nautobot_flags is not None:
            wrapper.nautobot.flags = nautobot_flags
        if pre_import:
            wrapper.pre_import = pre_import
        if disable_related_reference is not None:
            wrapper.disable_related_reference = disable_related_reference
        if forward_references:
            wrapper.forward_references = forward_references

        return wrapper

    def disable_model(self, content_type: ContentTypeStr, disable_reason: str) -> None:
        """Disable model importing."""
        self.get_or_create_wrapper(content_type).disable_reason = disable_reason

    def summarize(self, diffsync_summary: DiffSyncSummary) -> None:
        """Summarize the import."""
        self.summary.diffsync = diffsync_summary

        wrapper_to_id = {value: key for key, value in self.content_type_ids_mapping.items()}

        for content_type in sorted(self.wrappers):
            wrapper = self.wrappers.get(content_type)
            if wrapper:
                self.summary.source.append(wrapper.get_summary(wrapper_to_id.get(wrapper, None)))

        for content_type in sorted(self.nautobot.wrappers):
            wrapper = self.nautobot.wrappers.get(content_type)
            if wrapper:
                self.summary.nautobot.append(wrapper.get_summary())

    def get_or_create_wrapper(self, value: Union[None, SourceContentType]) -> "SourceModelWrapper":
        """Get a source Wrapper for a given content type."""
        # Enable mapping back from Nautobot content type, when using Nautobot model or wrapper
        map_back = False

        if not value:
            raise ValueError("Missing value")

        if isinstance(value, SourceModelWrapper):
            return value

        if isinstance(value, type(NautobotBaseModel)):
            map_back = True
            value = value._meta.label.lower()  # type: ignore
        elif isinstance(value, NautobotModelWrapper):
            map_back = True
            value = value.content_type

        if isinstance(value, str):
            value = value.lower()
        elif isinstance(value, int):
            if value not in self.content_type_ids_mapping:
                raise ValueError(f"Content type not found {value}")
            return self.content_type_ids_mapping[value]
        elif isinstance(value, Iterable) and len(value) == 2:  # noqa: PLR2004
            value = ".".join(value).lower()
        else:
            raise ValueError(f"Invalid content type {value}")

        if map_back and value in self._content_types_back_mapping:
            back_mapping = self._content_types_back_mapping.get(value, None)
            if not back_mapping:
                raise ValueError(f"Ambiguous content type back mapping {value}")
            value = back_mapping

        if value in self.wrappers:
            return self.wrappers[value]

        return self.configure_model(value)

    def get_nautobot_content_type_uid(self, content_type: ContentTypeValue) -> int:
        """Get the Django content type ID for a given content type."""
        if isinstance(content_type, int):
            wrapper = self.content_type_ids_mapping.get(content_type, None)
            if not wrapper:
                raise ValueError(f"Content type not found {content_type}")
            return wrapper.nautobot.content_type_instance.pk
        if not isinstance(content_type, str):
            if not len(content_type) == 2:
                raise ValueError(f"Invalid content type {content_type}")
            content_type = ".".join(content_type)

        wrapper = self.get_or_create_wrapper(content_type)

        return wrapper.nautobot.content_type_instance.pk

    def load(self) -> None:
        """Load data from the source."""
        self.import_data()
        self.post_import()

    def import_data(self) -> None:
        """Import data from the source."""
        get_source_data = self.get_source_data

        # First pass to enhance pre-defined wrappers structure
        for content_type, data in get_source_data():
            if content_type in self.wrappers:
                wrapper = self.wrappers[content_type]
            else:
                wrapper = self.configure_model(content_type)
            wrapper.first_pass(data)

        # Create importers, wrappers structure is updated as needed
        while True:
            wrappers = [
                wrapper
                for wrapper in self.wrappers.values()
                if wrapper.importers is None and not wrapper.disable_reason
            ]
            if not wrappers:
                break
            for wrapper in wrappers:
                wrapper.create_importers()

        # Second pass to import actual data
        for content_type, data in get_source_data():
            self.wrappers[content_type].second_pass(data)

    def post_import(self) -> None:
        """Post import processing."""
        while any(wrapper.post_import() for wrapper in self.wrappers.values()):
            pass

        for nautobot_wrapper in self.get_imported_nautobot_wrappers():
            diffsync_class = nautobot_wrapper.diffsync_class
            # pylint: disable=protected-access
            model_name = diffsync_class._modelname
            self.top_level.append(model_name)
            setattr(self, model_name, diffsync_class)
            setattr(self.nautobot, model_name, getattr(self, model_name))

    def get_imported_nautobot_wrappers(self) -> Generator[NautobotModelWrapper, None, None]:
        """Get a list of Nautobot model wrappers in the order of import."""
        result = OrderedDict()

        for wrapper in self.wrappers.values():
            if (
                wrapper
                and not wrapper.disable_reason
                and wrapper.stats.created > 0
                and wrapper.nautobot.content_type not in result
            ):
                result[wrapper.nautobot.content_type] = wrapper.nautobot

        for content_type in IMPORT_ORDER:
            if content_type in result:
                yield result[content_type]
                del result[content_type]

        yield from result.values()
__init__(*args, get_source_data, trace_issues=False, nautobot=None, logger=None, **kwargs)

Initialize the SourceAdapter.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(
    self,
    *args,
    get_source_data: SourceDataGenerator,
    trace_issues: bool = False,
    nautobot: Optional[NautobotAdapter] = None,
    logger=None,
    **kwargs,
):
    """Initialize the SourceAdapter."""
    super().__init__(*args, **kwargs)

    self.get_source_data = get_source_data
    self.wrappers: OrderedDict[ContentTypeStr, SourceModelWrapper] = OrderedDict()
    self.nautobot = nautobot or NautobotAdapter()
    self.nautobot.trace_issues = trace_issues
    self.content_type_ids_mapping: Dict[int, SourceModelWrapper] = {}
    self.logger = logger or default_logger
    self.summary = ImportSummary()

    # From Nautobot to Source content type mapping
    # When multiple source content types are mapped to the single nautobot content type, mapping is set to `None`
    self._content_types_back_mapping: Dict[ContentTypeStr, Optional[ContentTypeStr]] = {}
configure_model(content_type, nautobot_content_type='', extend_content_type='', identifiers=None, fields=None, default_reference=None, flags=None, nautobot_flags=None, pre_import=None, disable_related_reference=None, forward_references=None)

Create if not exist and configure a wrapper for a given source content type.

Create Nautobot content type wrapper as well.

Source code in nautobot_netbox_importer/generator/source.py
def configure_model(
    self,
    content_type: ContentTypeStr,
    nautobot_content_type: ContentTypeStr = "",
    extend_content_type: ContentTypeStr = "",
    identifiers: Optional[Iterable[FieldName]] = None,
    fields: Optional[Mapping[FieldName, SourceFieldDefinition]] = None,
    default_reference: Optional[RecordData] = None,
    flags: Optional[DiffSyncModelFlags] = None,
    nautobot_flags: Optional[DiffSyncModelFlags] = None,
    pre_import: Optional[PreImport] = None,
    disable_related_reference: Optional[bool] = None,
    forward_references: Optional[ForwardReferences] = None,
) -> "SourceModelWrapper":
    """Create if not exist and configure a wrapper for a given source content type.

    Create Nautobot content type wrapper as well.
    """
    content_type = content_type.lower()
    nautobot_content_type = nautobot_content_type.lower()
    extend_content_type = extend_content_type.lower()

    if extend_content_type:
        if nautobot_content_type:
            raise ValueError(f"Can't specify both nautobot_content_type and extend_content_type {content_type}")
        extends_wrapper = self.wrappers[extend_content_type]
        nautobot_content_type = extends_wrapper.nautobot.content_type
    else:
        extends_wrapper = None

    if content_type in self.wrappers:
        wrapper = self.wrappers[content_type]
        if nautobot_content_type and wrapper.nautobot.content_type != nautobot_content_type:
            raise ValueError(
                f"Content type {content_type} already mapped to {wrapper.nautobot.content_type} "
                f"can't map to {nautobot_content_type}"
            )
    else:
        nautobot_wrapper = self.nautobot.get_or_create_wrapper(nautobot_content_type or content_type)
        wrapper = SourceModelWrapper(self, content_type, nautobot_wrapper)
        if not extends_wrapper:
            if nautobot_wrapper.content_type in self._content_types_back_mapping:
                if self._content_types_back_mapping[nautobot_wrapper.content_type] != content_type:
                    self._content_types_back_mapping[nautobot_wrapper.content_type] = None
            else:
                self._content_types_back_mapping[nautobot_wrapper.content_type] = content_type

    if extends_wrapper:
        wrapper.extends_wrapper = extends_wrapper

    if identifiers:
        wrapper.set_identifiers(identifiers)
    for field_name, definition in (fields or {}).items():
        wrapper.add_field(field_name, SourceFieldSource.CUSTOM).set_definition(definition)
    if default_reference:
        wrapper.set_default_reference(default_reference)
    if flags is not None:
        wrapper.flags = flags
    if nautobot_flags is not None:
        wrapper.nautobot.flags = nautobot_flags
    if pre_import:
        wrapper.pre_import = pre_import
    if disable_related_reference is not None:
        wrapper.disable_related_reference = disable_related_reference
    if forward_references:
        wrapper.forward_references = forward_references

    return wrapper
disable_model(content_type, disable_reason)

Disable model importing.

Source code in nautobot_netbox_importer/generator/source.py
def disable_model(self, content_type: ContentTypeStr, disable_reason: str) -> None:
    """Disable model importing."""
    self.get_or_create_wrapper(content_type).disable_reason = disable_reason
get_imported_nautobot_wrappers()

Get a list of Nautobot model wrappers in the order of import.

Source code in nautobot_netbox_importer/generator/source.py
def get_imported_nautobot_wrappers(self) -> Generator[NautobotModelWrapper, None, None]:
    """Get a list of Nautobot model wrappers in the order of import."""
    result = OrderedDict()

    for wrapper in self.wrappers.values():
        if (
            wrapper
            and not wrapper.disable_reason
            and wrapper.stats.created > 0
            and wrapper.nautobot.content_type not in result
        ):
            result[wrapper.nautobot.content_type] = wrapper.nautobot

    for content_type in IMPORT_ORDER:
        if content_type in result:
            yield result[content_type]
            del result[content_type]

    yield from result.values()
get_nautobot_content_type_uid(content_type)

Get the Django content type ID for a given content type.

Source code in nautobot_netbox_importer/generator/source.py
def get_nautobot_content_type_uid(self, content_type: ContentTypeValue) -> int:
    """Get the Django content type ID for a given content type."""
    if isinstance(content_type, int):
        wrapper = self.content_type_ids_mapping.get(content_type, None)
        if not wrapper:
            raise ValueError(f"Content type not found {content_type}")
        return wrapper.nautobot.content_type_instance.pk
    if not isinstance(content_type, str):
        if not len(content_type) == 2:
            raise ValueError(f"Invalid content type {content_type}")
        content_type = ".".join(content_type)

    wrapper = self.get_or_create_wrapper(content_type)

    return wrapper.nautobot.content_type_instance.pk
get_or_create_wrapper(value)

Get a source Wrapper for a given content type.

Source code in nautobot_netbox_importer/generator/source.py
def get_or_create_wrapper(self, value: Union[None, SourceContentType]) -> "SourceModelWrapper":
    """Get a source Wrapper for a given content type."""
    # Enable mapping back from Nautobot content type, when using Nautobot model or wrapper
    map_back = False

    if not value:
        raise ValueError("Missing value")

    if isinstance(value, SourceModelWrapper):
        return value

    if isinstance(value, type(NautobotBaseModel)):
        map_back = True
        value = value._meta.label.lower()  # type: ignore
    elif isinstance(value, NautobotModelWrapper):
        map_back = True
        value = value.content_type

    if isinstance(value, str):
        value = value.lower()
    elif isinstance(value, int):
        if value not in self.content_type_ids_mapping:
            raise ValueError(f"Content type not found {value}")
        return self.content_type_ids_mapping[value]
    elif isinstance(value, Iterable) and len(value) == 2:  # noqa: PLR2004
        value = ".".join(value).lower()
    else:
        raise ValueError(f"Invalid content type {value}")

    if map_back and value in self._content_types_back_mapping:
        back_mapping = self._content_types_back_mapping.get(value, None)
        if not back_mapping:
            raise ValueError(f"Ambiguous content type back mapping {value}")
        value = back_mapping

    if value in self.wrappers:
        return self.wrappers[value]

    return self.configure_model(value)
import_data()

Import data from the source.

Source code in nautobot_netbox_importer/generator/source.py
def import_data(self) -> None:
    """Import data from the source."""
    get_source_data = self.get_source_data

    # First pass to enhance pre-defined wrappers structure
    for content_type, data in get_source_data():
        if content_type in self.wrappers:
            wrapper = self.wrappers[content_type]
        else:
            wrapper = self.configure_model(content_type)
        wrapper.first_pass(data)

    # Create importers, wrappers structure is updated as needed
    while True:
        wrappers = [
            wrapper
            for wrapper in self.wrappers.values()
            if wrapper.importers is None and not wrapper.disable_reason
        ]
        if not wrappers:
            break
        for wrapper in wrappers:
            wrapper.create_importers()

    # Second pass to import actual data
    for content_type, data in get_source_data():
        self.wrappers[content_type].second_pass(data)
load()

Load data from the source.

Source code in nautobot_netbox_importer/generator/source.py
def load(self) -> None:
    """Load data from the source."""
    self.import_data()
    self.post_import()
post_import()

Post import processing.

Source code in nautobot_netbox_importer/generator/source.py
def post_import(self) -> None:
    """Post import processing."""
    while any(wrapper.post_import() for wrapper in self.wrappers.values()):
        pass

    for nautobot_wrapper in self.get_imported_nautobot_wrappers():
        diffsync_class = nautobot_wrapper.diffsync_class
        # pylint: disable=protected-access
        model_name = diffsync_class._modelname
        self.top_level.append(model_name)
        setattr(self, model_name, diffsync_class)
        setattr(self.nautobot, model_name, getattr(self, model_name))
summarize(diffsync_summary)

Summarize the import.

Source code in nautobot_netbox_importer/generator/source.py
def summarize(self, diffsync_summary: DiffSyncSummary) -> None:
    """Summarize the import."""
    self.summary.diffsync = diffsync_summary

    wrapper_to_id = {value: key for key, value in self.content_type_ids_mapping.items()}

    for content_type in sorted(self.wrappers):
        wrapper = self.wrappers.get(content_type)
        if wrapper:
            self.summary.source.append(wrapper.get_summary(wrapper_to_id.get(wrapper, None)))

    for content_type in sorted(self.nautobot.wrappers):
        wrapper = self.nautobot.wrappers.get(content_type)
        if wrapper:
            self.summary.nautobot.append(wrapper.get_summary())
SourceField

Source Field.

Source code in nautobot_netbox_importer/generator/source.py
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
class SourceField:
    """Source Field."""

    def __init__(self, wrapper: SourceModelWrapper, name: FieldName, source: SourceFieldSource):
        """Initialize the SourceField."""
        self.wrapper = wrapper
        wrapper.fields[name] = self
        self.name = name
        self.definition: SourceFieldDefinition = name
        self.sources = set((source,))
        self.processed = False
        self._nautobot: Optional[NautobotField] = None
        self.importer: Optional[SourceFieldImporter] = None
        self.default_value: Any = None
        self.disable_reason: str = ""

    def __str__(self) -> str:
        """Return a string representation of the field."""
        return self.wrapper.format_field_name(self.name)

    @property
    def nautobot(self) -> NautobotField:
        """Get the Nautobot field wrapper."""
        if not self._nautobot:
            raise RuntimeError(f"Missing Nautobot field for {self}")
        return self._nautobot

    def get_summary(self) -> FieldSummary:
        """Get a summary of the field."""
        return FieldSummary(
            name=self.name,
            nautobot_name=self._nautobot and self._nautobot.name,
            nautobot_internal_type=self._nautobot and self._nautobot.internal_type.value,
            nautobot_can_import=self._nautobot and self._nautobot.can_import,
            importer=self.importer and self.importer.__name__,
            definition=serialize_to_summary(self.definition),
            sources=sorted(source.name for source in self.sources),
            default_value=serialize_to_summary(self.default_value),
            disable_reason=self.disable_reason,
            required=self._nautobot.required if self._nautobot else False,
        )

    def disable(self, reason: str) -> None:
        """Disable field importing."""
        self.definition = None
        self.importer = None
        self.processed = True
        self.disable_reason = reason

    def handle_sibling(self, sibling: Union["SourceField", FieldName], nautobot_name: FieldName = "") -> "SourceField":
        """Specify, that this field importer handles other field."""
        if not self.importer:
            raise RuntimeError(f"Call `handle sibling` after setting importer for {self}")

        if isinstance(sibling, FieldName):
            sibling = self.wrapper.add_field(sibling, SourceFieldSource.SIBLING)

        sibling.set_nautobot_field(nautobot_name or self.nautobot.name)
        sibling.importer = self.importer
        sibling.processed = True

        if self.nautobot.can_import and not sibling.nautobot.can_import:
            self.disable(f"Can't import {self} based on {sibling}")

        return sibling

    def add_issue(self, issue_type: str, message: str, target: Optional[DiffSyncModel] = None) -> None:
        """Add an importer issue to the Nautobot Model Wrapper."""
        self.wrapper.nautobot.add_issue(issue_type, message=str({self.name: message}), diffsync_instance=target)

    def set_definition(self, definition: SourceFieldDefinition) -> None:
        """Customize field definition."""
        if self.processed:
            raise RuntimeError(f"Field already processed. {self}")

        if self.definition != definition:
            if self.definition != self.name:
                self.add_issue(
                    "OverrideDefinition",
                    f"Overriding field definition | Original: `{self.definition}` | New: `{definition}`",
                )
            self.definition = definition

    def create_importer(self) -> None:
        """Create importer for the field."""
        if self.processed:
            return
        self.processed = True

        if self.definition is None:
            return

        if isinstance(self.definition, FieldName):
            self.set_importer(nautobot_name=self.definition)
        elif callable(self.definition):
            self.definition(self)
        else:
            raise NotImplementedError(f"Unsupported field definition {self.definition}")

    def get_source_value(self, source: RecordData) -> Any:
        """Get a value from the source data, returning a default value if the value is empty."""
        if self.name not in source:
            return self.default_value

        result = source[self.name]
        return self.default_value if result in EMPTY_VALUES else result

    def set_nautobot_value(self, target: DiffSyncModel, value: Any) -> None:
        """Set a value to the Nautobot model."""
        if value in EMPTY_VALUES:
            if hasattr(target, self.nautobot.name):
                delattr(target, self.nautobot.name)
        else:
            setattr(target, self.nautobot.name, value)

    def set_nautobot_field(self, nautobot_name: FieldName = "") -> NautobotField:
        """Set a Nautobot field name for the field."""
        result = self.wrapper.nautobot.add_field(nautobot_name or self.name)
        if result.field:
            default_value = getattr(result.field, "default", None)
            if default_value not in EMPTY_VALUES and not isinstance(default_value, Callable):
                self.default_value = default_value
        self._nautobot = result
        if result.name == "last_updated":
            self.disable("Last updated field is updated with each write")
        return result

    # pylint: disable=too-many-branches
    def set_importer(
        self,
        importer: Optional[SourceFieldImporter] = None,
        nautobot_name: Optional[FieldName] = "",
        override=False,
    ) -> Optional[SourceFieldImporter]:
        """Sets the importer and Nautobot field if not already specified.

        If `nautobot_name` is not provided, the field name is used.

        Passing None to `nautobot_name` indicates that there is custom mapping without a direct relationship to a Nautobot field.
        """
        if self.disable_reason:
            raise RuntimeError(f"Can't set importer for disabled {self}")
        if self.importer and not override:
            raise RuntimeError(f"Importer already set for {self}")
        if not self._nautobot and nautobot_name is not None:
            self.set_nautobot_field(nautobot_name)

        if importer:
            self.importer = importer
            return importer

        if self.disable_reason or not self.nautobot.can_import:
            return None

        internal_type = self.nautobot.internal_type

        if internal_type == InternalFieldType.JSON_FIELD:
            self.set_json_importer()
        elif internal_type == InternalFieldType.DATE_FIELD:
            self.set_date_importer()
        elif internal_type == InternalFieldType.DATE_TIME_FIELD:
            self.set_datetime_importer()
        elif internal_type == InternalFieldType.UUID_FIELD:
            self.set_uuid_importer()
        elif internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
            self.set_m2m_importer()
        elif internal_type == InternalFieldType.STATUS_FIELD:
            self.set_status_importer()
        elif self.nautobot.is_reference:
            self.set_relation_importer()
        elif getattr(self.nautobot.field, "choices", None):
            self.set_choice_importer()
        elif self.nautobot.is_integer:
            self.set_integer_importer()
        else:
            self.set_value_importer()

        return self.importer

    def set_value_importer(self) -> None:
        """Set a value importer."""

        def value_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = self.get_source_value(source)
            self.set_nautobot_value(target, value)

        self.set_importer(value_importer)

    def set_json_importer(self) -> None:
        """Set a JSON field importer."""

        def json_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(self.name, None)
            if isinstance(value, str) and value:
                value = json.loads(value)
            self.set_nautobot_value(target, value)

        self.set_importer(json_importer)

    def set_choice_importer(self) -> None:
        """Set a choice field importer."""
        field_choices = getattr(self.nautobot.field, "choices", None)
        if not field_choices:
            raise ValueError(f"Invalid field_choices for {self}")

        choices = dict(get_field_choices(field_choices))

        def choice_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = self.get_source_value(source)
            if value in choices:
                self.set_nautobot_value(target, value)
            elif self.nautobot.required:
                # Set the choice value even it's not valid in Nautobot as it's required
                self.set_nautobot_value(target, value)
                raise InvalidChoiceValueIssue(self, value)
            elif value in EMPTY_VALUES:
                self.set_nautobot_value(target, value)
            else:
                raise InvalidChoiceValueIssue(self, value, None)

        self.set_importer(choice_importer)

    def set_integer_importer(self) -> None:
        """Set an integer field importer."""

        def integer_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            source_value = self.get_source_value(source)
            if source_value in EMPTY_VALUES:
                self.set_nautobot_value(target, source_value)
            else:
                source_value = float(source_value)
                value = int(source_value)
                self.set_nautobot_value(target, value)
                if value != source_value:
                    raise SourceFieldImporterIssue(f"Invalid source value {source_value}, truncated to {value}", self)

        self.set_importer(integer_importer)

    def _get_related_wrapper(self, related_source: Optional[SourceContentType]) -> SourceModelWrapper:
        """Get a related wrapper."""
        if related_source:
            return self.wrapper.adapter.get_or_create_wrapper(related_source)

        if self.name == "parent":
            return self.wrapper

        return self.wrapper.adapter.get_or_create_wrapper(self.nautobot.related_model)

    def set_relation_importer(self, related_source: Optional[SourceContentType] = None) -> None:
        """Set a relation importer."""
        related_wrapper = self._get_related_wrapper(related_source)

        if self.nautobot.is_content_type:
            self.set_content_type_importer()
            return

        if self.default_value in EMPTY_VALUES and related_wrapper.default_reference_uid:
            self.default_value = related_wrapper.default_reference_uid

        if not (self.default_value is None or isinstance(self.default_value, UUID)):
            raise NotImplementedError(f"Default value {self.default_value} is not a UUID")

        def relation_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = self.get_source_value(source)
            if value in EMPTY_VALUES:
                self.set_nautobot_value(target, value)
            else:
                if isinstance(value, (UUID, str, int)):
                    result = related_wrapper.get_pk_from_uid(value)
                else:
                    result = related_wrapper.get_pk_from_identifiers(value)
                self.set_nautobot_value(target, result)
                self.wrapper.add_reference(related_wrapper, result)

        self.set_importer(relation_importer)

    def set_content_type_importer(self) -> None:
        """Set a content type importer."""
        adapter = self.wrapper.adapter

        def content_type_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            content_type = source.get(self.name, None)
            if content_type not in EMPTY_VALUES:
                content_type = adapter.get_nautobot_content_type_uid(content_type)
            self.set_nautobot_value(target, content_type)

        self.set_importer(content_type_importer)

    def set_m2m_importer(self, related_source: Optional[SourceContentType] = None) -> None:
        """Set a many to many importer."""
        if not isinstance(self.nautobot.field, DjangoField):
            raise NotImplementedError(f"Unsupported m2m importer {self}")

        related_wrapper = self._get_related_wrapper(related_source)

        if related_wrapper.content_type == "contenttypes.contenttype":
            self.set_content_types_importer()
            return

        def m2m_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            values = source.get(self.name, None)
            if values in EMPTY_VALUES:
                return

            if isinstance(values, (UUID, str, int)):
                result = related_wrapper.get_pk_from_uid(values)
                self.wrapper.add_reference(related_wrapper, result)
                self.set_nautobot_value(target, {result})
                return

            if not isinstance(values, (list, set)):
                raise ValueError(f"Invalid value {values} for field {self.name}")

            results = set()
            for value in values:
                if isinstance(value, (UUID, str, int)):
                    result = related_wrapper.get_pk_from_uid(value)
                else:
                    result = related_wrapper.get_pk_from_identifiers(value)

                results.add(result)
                self.wrapper.add_reference(related_wrapper, result)

            self.set_nautobot_value(target, results)

        self.set_importer(m2m_importer)

    def set_content_types_importer(self) -> None:
        """Set a content types importer."""
        adapter = self.wrapper.adapter

        def content_types_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            values = source.get(self.name, None)
            if values in EMPTY_VALUES:
                return

            if not isinstance(values, (list, set)):
                raise ValueError(f"Invalid value {values} for field {self.name}")

            nautobot_values = set()
            for item in values:
                try:
                    nautobot_values.add(adapter.get_nautobot_content_type_uid(item))
                except NautobotModelNotFound:
                    self.add_issue("InvalidContentType", f"Invalid content type {item}, skipping", target)

            self.set_nautobot_value(target, nautobot_values)

        self.set_importer(content_types_importer)

    def set_datetime_importer(self) -> None:
        """Set a datetime importer."""

        def datetime_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(self.name, None)
            if value not in EMPTY_VALUES:
                value = normalize_datetime(value)
            self.set_nautobot_value(target, value)

        self.set_importer(datetime_importer)

    def set_relation_and_type_importer(self, type_field: "SourceField") -> None:
        """Set a relation UUID importer based on the type field."""

        def relation_and_type_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            source_uid = source.get(self.name, None)
            source_type = source.get(type_field.name, None)
            if source_type in EMPTY_VALUES or source_uid in EMPTY_VALUES:
                if source_uid not in EMPTY_VALUES or source_type not in EMPTY_VALUES:
                    raise ValueError(
                        f"Both {self}=`{source_uid}` and {type_field}=`{source_type}` must be empty or not empty."
                    )
                return

            type_wrapper = self.wrapper.adapter.get_or_create_wrapper(source_type)
            uid = type_wrapper.get_pk_from_uid(source_uid)
            self.set_nautobot_value(target, uid)
            type_field.set_nautobot_value(target, type_wrapper.nautobot.content_type_instance.pk)
            self.wrapper.add_reference(type_wrapper, uid)

        self.set_importer(relation_and_type_importer)
        self.handle_sibling(type_field, type_field.name)

    def set_uuid_importer(self) -> None:
        """Set an UUID importer."""
        if self.name.endswith("_id"):
            type_field = self.wrapper.fields.get(self.name[:-3] + "_type", None)
            if type_field and type_field.nautobot.is_content_type:
                # Handles `<field name>_id` and `<field name>_type` fields combination
                self.set_relation_and_type_importer(type_field)
                return

        def uuid_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(self.name, None)
            if value not in EMPTY_VALUES:
                value = UUID(value)
            self.set_nautobot_value(target, value)

        self.set_importer(uuid_importer)

    def set_date_importer(self) -> None:
        """Set a date importer."""

        def date_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            value = source.get(self.name, None)
            if value not in EMPTY_VALUES and not isinstance(value, datetime.date):
                value = datetime.date.fromisoformat(str(value))
            self.set_nautobot_value(target, value)

        self.set_importer(date_importer)

    def set_status_importer(self) -> None:
        """Set a status importer."""
        status_wrapper = self.wrapper.adapter.get_or_create_wrapper("extras.status")
        if not self.default_value:
            self.default_value = status_wrapper.default_reference_uid

        if not (self.default_value is None or isinstance(self.default_value, UUID)):
            raise NotImplementedError(f"Default value {self.default_value} is not a UUID")

        def status_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
            status = source.get(self.name, None)
            if status:
                value = status_wrapper.cache_record({"name": status[0].upper() + status[1:]})
            else:
                value = self.default_value

            self.set_nautobot_value(target, value)
            if value:
                self.wrapper.add_reference(status_wrapper, value)

        self.set_importer(status_importer)
nautobot: NautobotField property

Get the Nautobot field wrapper.

__init__(wrapper, name, source)

Initialize the SourceField.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(self, wrapper: SourceModelWrapper, name: FieldName, source: SourceFieldSource):
    """Initialize the SourceField."""
    self.wrapper = wrapper
    wrapper.fields[name] = self
    self.name = name
    self.definition: SourceFieldDefinition = name
    self.sources = set((source,))
    self.processed = False
    self._nautobot: Optional[NautobotField] = None
    self.importer: Optional[SourceFieldImporter] = None
    self.default_value: Any = None
    self.disable_reason: str = ""
__str__()

Return a string representation of the field.

Source code in nautobot_netbox_importer/generator/source.py
def __str__(self) -> str:
    """Return a string representation of the field."""
    return self.wrapper.format_field_name(self.name)
add_issue(issue_type, message, target=None)

Add an importer issue to the Nautobot Model Wrapper.

Source code in nautobot_netbox_importer/generator/source.py
def add_issue(self, issue_type: str, message: str, target: Optional[DiffSyncModel] = None) -> None:
    """Add an importer issue to the Nautobot Model Wrapper."""
    self.wrapper.nautobot.add_issue(issue_type, message=str({self.name: message}), diffsync_instance=target)
create_importer()

Create importer for the field.

Source code in nautobot_netbox_importer/generator/source.py
def create_importer(self) -> None:
    """Create importer for the field."""
    if self.processed:
        return
    self.processed = True

    if self.definition is None:
        return

    if isinstance(self.definition, FieldName):
        self.set_importer(nautobot_name=self.definition)
    elif callable(self.definition):
        self.definition(self)
    else:
        raise NotImplementedError(f"Unsupported field definition {self.definition}")
disable(reason)

Disable field importing.

Source code in nautobot_netbox_importer/generator/source.py
def disable(self, reason: str) -> None:
    """Disable field importing."""
    self.definition = None
    self.importer = None
    self.processed = True
    self.disable_reason = reason
get_source_value(source)

Get a value from the source data, returning a default value if the value is empty.

Source code in nautobot_netbox_importer/generator/source.py
def get_source_value(self, source: RecordData) -> Any:
    """Get a value from the source data, returning a default value if the value is empty."""
    if self.name not in source:
        return self.default_value

    result = source[self.name]
    return self.default_value if result in EMPTY_VALUES else result
get_summary()

Get a summary of the field.

Source code in nautobot_netbox_importer/generator/source.py
def get_summary(self) -> FieldSummary:
    """Get a summary of the field."""
    return FieldSummary(
        name=self.name,
        nautobot_name=self._nautobot and self._nautobot.name,
        nautobot_internal_type=self._nautobot and self._nautobot.internal_type.value,
        nautobot_can_import=self._nautobot and self._nautobot.can_import,
        importer=self.importer and self.importer.__name__,
        definition=serialize_to_summary(self.definition),
        sources=sorted(source.name for source in self.sources),
        default_value=serialize_to_summary(self.default_value),
        disable_reason=self.disable_reason,
        required=self._nautobot.required if self._nautobot else False,
    )
handle_sibling(sibling, nautobot_name='')

Specify, that this field importer handles other field.

Source code in nautobot_netbox_importer/generator/source.py
def handle_sibling(self, sibling: Union["SourceField", FieldName], nautobot_name: FieldName = "") -> "SourceField":
    """Specify, that this field importer handles other field."""
    if not self.importer:
        raise RuntimeError(f"Call `handle sibling` after setting importer for {self}")

    if isinstance(sibling, FieldName):
        sibling = self.wrapper.add_field(sibling, SourceFieldSource.SIBLING)

    sibling.set_nautobot_field(nautobot_name or self.nautobot.name)
    sibling.importer = self.importer
    sibling.processed = True

    if self.nautobot.can_import and not sibling.nautobot.can_import:
        self.disable(f"Can't import {self} based on {sibling}")

    return sibling
set_choice_importer()

Set a choice field importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_choice_importer(self) -> None:
    """Set a choice field importer."""
    field_choices = getattr(self.nautobot.field, "choices", None)
    if not field_choices:
        raise ValueError(f"Invalid field_choices for {self}")

    choices = dict(get_field_choices(field_choices))

    def choice_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = self.get_source_value(source)
        if value in choices:
            self.set_nautobot_value(target, value)
        elif self.nautobot.required:
            # Set the choice value even it's not valid in Nautobot as it's required
            self.set_nautobot_value(target, value)
            raise InvalidChoiceValueIssue(self, value)
        elif value in EMPTY_VALUES:
            self.set_nautobot_value(target, value)
        else:
            raise InvalidChoiceValueIssue(self, value, None)

    self.set_importer(choice_importer)
set_content_type_importer()

Set a content type importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_content_type_importer(self) -> None:
    """Set a content type importer."""
    adapter = self.wrapper.adapter

    def content_type_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        content_type = source.get(self.name, None)
        if content_type not in EMPTY_VALUES:
            content_type = adapter.get_nautobot_content_type_uid(content_type)
        self.set_nautobot_value(target, content_type)

    self.set_importer(content_type_importer)
set_content_types_importer()

Set a content types importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_content_types_importer(self) -> None:
    """Set a content types importer."""
    adapter = self.wrapper.adapter

    def content_types_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        values = source.get(self.name, None)
        if values in EMPTY_VALUES:
            return

        if not isinstance(values, (list, set)):
            raise ValueError(f"Invalid value {values} for field {self.name}")

        nautobot_values = set()
        for item in values:
            try:
                nautobot_values.add(adapter.get_nautobot_content_type_uid(item))
            except NautobotModelNotFound:
                self.add_issue("InvalidContentType", f"Invalid content type {item}, skipping", target)

        self.set_nautobot_value(target, nautobot_values)

    self.set_importer(content_types_importer)
set_date_importer()

Set a date importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_date_importer(self) -> None:
    """Set a date importer."""

    def date_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = source.get(self.name, None)
        if value not in EMPTY_VALUES and not isinstance(value, datetime.date):
            value = datetime.date.fromisoformat(str(value))
        self.set_nautobot_value(target, value)

    self.set_importer(date_importer)
set_datetime_importer()

Set a datetime importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_datetime_importer(self) -> None:
    """Set a datetime importer."""

    def datetime_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = source.get(self.name, None)
        if value not in EMPTY_VALUES:
            value = normalize_datetime(value)
        self.set_nautobot_value(target, value)

    self.set_importer(datetime_importer)
set_definition(definition)

Customize field definition.

Source code in nautobot_netbox_importer/generator/source.py
def set_definition(self, definition: SourceFieldDefinition) -> None:
    """Customize field definition."""
    if self.processed:
        raise RuntimeError(f"Field already processed. {self}")

    if self.definition != definition:
        if self.definition != self.name:
            self.add_issue(
                "OverrideDefinition",
                f"Overriding field definition | Original: `{self.definition}` | New: `{definition}`",
            )
        self.definition = definition
set_importer(importer=None, nautobot_name='', override=False)

Sets the importer and Nautobot field if not already specified.

If nautobot_name is not provided, the field name is used.

Passing None to nautobot_name indicates that there is custom mapping without a direct relationship to a Nautobot field.

Source code in nautobot_netbox_importer/generator/source.py
def set_importer(
    self,
    importer: Optional[SourceFieldImporter] = None,
    nautobot_name: Optional[FieldName] = "",
    override=False,
) -> Optional[SourceFieldImporter]:
    """Sets the importer and Nautobot field if not already specified.

    If `nautobot_name` is not provided, the field name is used.

    Passing None to `nautobot_name` indicates that there is custom mapping without a direct relationship to a Nautobot field.
    """
    if self.disable_reason:
        raise RuntimeError(f"Can't set importer for disabled {self}")
    if self.importer and not override:
        raise RuntimeError(f"Importer already set for {self}")
    if not self._nautobot and nautobot_name is not None:
        self.set_nautobot_field(nautobot_name)

    if importer:
        self.importer = importer
        return importer

    if self.disable_reason or not self.nautobot.can_import:
        return None

    internal_type = self.nautobot.internal_type

    if internal_type == InternalFieldType.JSON_FIELD:
        self.set_json_importer()
    elif internal_type == InternalFieldType.DATE_FIELD:
        self.set_date_importer()
    elif internal_type == InternalFieldType.DATE_TIME_FIELD:
        self.set_datetime_importer()
    elif internal_type == InternalFieldType.UUID_FIELD:
        self.set_uuid_importer()
    elif internal_type == InternalFieldType.MANY_TO_MANY_FIELD:
        self.set_m2m_importer()
    elif internal_type == InternalFieldType.STATUS_FIELD:
        self.set_status_importer()
    elif self.nautobot.is_reference:
        self.set_relation_importer()
    elif getattr(self.nautobot.field, "choices", None):
        self.set_choice_importer()
    elif self.nautobot.is_integer:
        self.set_integer_importer()
    else:
        self.set_value_importer()

    return self.importer
set_integer_importer()

Set an integer field importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_integer_importer(self) -> None:
    """Set an integer field importer."""

    def integer_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        source_value = self.get_source_value(source)
        if source_value in EMPTY_VALUES:
            self.set_nautobot_value(target, source_value)
        else:
            source_value = float(source_value)
            value = int(source_value)
            self.set_nautobot_value(target, value)
            if value != source_value:
                raise SourceFieldImporterIssue(f"Invalid source value {source_value}, truncated to {value}", self)

    self.set_importer(integer_importer)
set_json_importer()

Set a JSON field importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_json_importer(self) -> None:
    """Set a JSON field importer."""

    def json_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = source.get(self.name, None)
        if isinstance(value, str) and value:
            value = json.loads(value)
        self.set_nautobot_value(target, value)

    self.set_importer(json_importer)
set_m2m_importer(related_source=None)

Set a many to many importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_m2m_importer(self, related_source: Optional[SourceContentType] = None) -> None:
    """Set a many to many importer."""
    if not isinstance(self.nautobot.field, DjangoField):
        raise NotImplementedError(f"Unsupported m2m importer {self}")

    related_wrapper = self._get_related_wrapper(related_source)

    if related_wrapper.content_type == "contenttypes.contenttype":
        self.set_content_types_importer()
        return

    def m2m_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        values = source.get(self.name, None)
        if values in EMPTY_VALUES:
            return

        if isinstance(values, (UUID, str, int)):
            result = related_wrapper.get_pk_from_uid(values)
            self.wrapper.add_reference(related_wrapper, result)
            self.set_nautobot_value(target, {result})
            return

        if not isinstance(values, (list, set)):
            raise ValueError(f"Invalid value {values} for field {self.name}")

        results = set()
        for value in values:
            if isinstance(value, (UUID, str, int)):
                result = related_wrapper.get_pk_from_uid(value)
            else:
                result = related_wrapper.get_pk_from_identifiers(value)

            results.add(result)
            self.wrapper.add_reference(related_wrapper, result)

        self.set_nautobot_value(target, results)

    self.set_importer(m2m_importer)
set_nautobot_field(nautobot_name='')

Set a Nautobot field name for the field.

Source code in nautobot_netbox_importer/generator/source.py
def set_nautobot_field(self, nautobot_name: FieldName = "") -> NautobotField:
    """Set a Nautobot field name for the field."""
    result = self.wrapper.nautobot.add_field(nautobot_name or self.name)
    if result.field:
        default_value = getattr(result.field, "default", None)
        if default_value not in EMPTY_VALUES and not isinstance(default_value, Callable):
            self.default_value = default_value
    self._nautobot = result
    if result.name == "last_updated":
        self.disable("Last updated field is updated with each write")
    return result
set_nautobot_value(target, value)

Set a value to the Nautobot model.

Source code in nautobot_netbox_importer/generator/source.py
def set_nautobot_value(self, target: DiffSyncModel, value: Any) -> None:
    """Set a value to the Nautobot model."""
    if value in EMPTY_VALUES:
        if hasattr(target, self.nautobot.name):
            delattr(target, self.nautobot.name)
    else:
        setattr(target, self.nautobot.name, value)
set_relation_and_type_importer(type_field)

Set a relation UUID importer based on the type field.

Source code in nautobot_netbox_importer/generator/source.py
def set_relation_and_type_importer(self, type_field: "SourceField") -> None:
    """Set a relation UUID importer based on the type field."""

    def relation_and_type_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        source_uid = source.get(self.name, None)
        source_type = source.get(type_field.name, None)
        if source_type in EMPTY_VALUES or source_uid in EMPTY_VALUES:
            if source_uid not in EMPTY_VALUES or source_type not in EMPTY_VALUES:
                raise ValueError(
                    f"Both {self}=`{source_uid}` and {type_field}=`{source_type}` must be empty or not empty."
                )
            return

        type_wrapper = self.wrapper.adapter.get_or_create_wrapper(source_type)
        uid = type_wrapper.get_pk_from_uid(source_uid)
        self.set_nautobot_value(target, uid)
        type_field.set_nautobot_value(target, type_wrapper.nautobot.content_type_instance.pk)
        self.wrapper.add_reference(type_wrapper, uid)

    self.set_importer(relation_and_type_importer)
    self.handle_sibling(type_field, type_field.name)
set_relation_importer(related_source=None)

Set a relation importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_relation_importer(self, related_source: Optional[SourceContentType] = None) -> None:
    """Set a relation importer."""
    related_wrapper = self._get_related_wrapper(related_source)

    if self.nautobot.is_content_type:
        self.set_content_type_importer()
        return

    if self.default_value in EMPTY_VALUES and related_wrapper.default_reference_uid:
        self.default_value = related_wrapper.default_reference_uid

    if not (self.default_value is None or isinstance(self.default_value, UUID)):
        raise NotImplementedError(f"Default value {self.default_value} is not a UUID")

    def relation_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = self.get_source_value(source)
        if value in EMPTY_VALUES:
            self.set_nautobot_value(target, value)
        else:
            if isinstance(value, (UUID, str, int)):
                result = related_wrapper.get_pk_from_uid(value)
            else:
                result = related_wrapper.get_pk_from_identifiers(value)
            self.set_nautobot_value(target, result)
            self.wrapper.add_reference(related_wrapper, result)

    self.set_importer(relation_importer)
set_status_importer()

Set a status importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_status_importer(self) -> None:
    """Set a status importer."""
    status_wrapper = self.wrapper.adapter.get_or_create_wrapper("extras.status")
    if not self.default_value:
        self.default_value = status_wrapper.default_reference_uid

    if not (self.default_value is None or isinstance(self.default_value, UUID)):
        raise NotImplementedError(f"Default value {self.default_value} is not a UUID")

    def status_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        status = source.get(self.name, None)
        if status:
            value = status_wrapper.cache_record({"name": status[0].upper() + status[1:]})
        else:
            value = self.default_value

        self.set_nautobot_value(target, value)
        if value:
            self.wrapper.add_reference(status_wrapper, value)

    self.set_importer(status_importer)
set_uuid_importer()

Set an UUID importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_uuid_importer(self) -> None:
    """Set an UUID importer."""
    if self.name.endswith("_id"):
        type_field = self.wrapper.fields.get(self.name[:-3] + "_type", None)
        if type_field and type_field.nautobot.is_content_type:
            # Handles `<field name>_id` and `<field name>_type` fields combination
            self.set_relation_and_type_importer(type_field)
            return

    def uuid_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = source.get(self.name, None)
        if value not in EMPTY_VALUES:
            value = UUID(value)
        self.set_nautobot_value(target, value)

    self.set_importer(uuid_importer)
set_value_importer()

Set a value importer.

Source code in nautobot_netbox_importer/generator/source.py
def set_value_importer(self) -> None:
    """Set a value importer."""

    def value_importer(source: RecordData, target: DiffSyncBaseModel) -> None:
        value = self.get_source_value(source)
        self.set_nautobot_value(target, value)

    self.set_importer(value_importer)
SourceFieldImporterIssue

Bases: NetBoxImporterException

Raised when an error occurs during field import.

Source code in nautobot_netbox_importer/generator/source.py
class SourceFieldImporterIssue(NetBoxImporterException):
    """Raised when an error occurs during field import."""

    def __init__(self, message: str, field: "SourceField"):
        """Initialize the exception."""
        super().__init__(str({field.name: message}))
__init__(message, field)

Initialize the exception.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(self, message: str, field: "SourceField"):
    """Initialize the exception."""
    super().__init__(str({field.name: message}))
SourceFieldSource

Bases: Enum

Defines the source of the SourceField.

Source code in nautobot_netbox_importer/generator/source.py
class SourceFieldSource(Enum):
    """Defines the source of the SourceField."""

    AUTO = auto()  # Automatically added fields like primary keys and AUTO_ADD_FIELDS
    CACHE = auto()  # Fields added by caching data during customization
    DATA = auto()  # Fields added from input data
    CUSTOM = auto()  # Fields added by customizing the importer
    SIBLING = auto()  # Fields defined as siblings of other fields, imported by other field importer
    IDENTIFIER = auto()  # Fields used as identifiers
SourceModelWrapper

Definition of a source model mapping to Nautobot model.

Source code in nautobot_netbox_importer/generator/source.py
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
class SourceModelWrapper:
    """Definition of a source model mapping to Nautobot model."""

    def __init__(self, adapter: SourceAdapter, content_type: ContentTypeStr, nautobot_wrapper: NautobotModelWrapper):
        """Initialize the SourceModelWrapper."""
        if content_type in adapter.wrappers:
            raise ValueError(f"Duplicate content type {content_type}")
        adapter.wrappers[content_type] = self
        self.adapter = adapter
        self.content_type = content_type
        self.nautobot = nautobot_wrapper
        if self.nautobot.disabled:
            self.disable_reason = f"Nautobot content type: `{nautobot_wrapper.content_type}` not found"
        else:
            self.disable_reason = ""

        # Source field names when referencing this model
        self.identifiers: Optional[List[FieldName]] = None

        # Used to autofill `content_types` field
        self.disable_related_reference = False
        self.references: SourceReferences = {}
        self.forward_references: Optional[ForwardReferences] = None

        # Whether importing record data exteds existing record
        self.extends_wrapper: Optional[SourceModelWrapper] = None

        # Importers are created after all fields are defined
        self.importers: Optional[Set[SourceFieldImporter]] = None

        # Default reference to this model
        self.default_reference_uid: Optional[Uid] = None

        # Caching
        self._uid_to_pk_cache: Dict[Uid, Uid] = {}
        self._cached_data: Dict[Uid, RecordData] = {}

        self.stats = SourceModelStats()
        self.flags = DiffSyncModelFlags.NONE

        # Source fields defintions
        self.fields: OrderedDict[FieldName, SourceField] = OrderedDict()
        self.pre_import: Optional[PreImport] = None

        if self.disable_reason:
            self.adapter.logger.debug("Created disabled %s", self)
            return

        pk_field = self.add_field(nautobot_wrapper.pk_field.name, SourceFieldSource.AUTO)
        pk_field.set_nautobot_field()
        pk_field.processed = True

        if issubclass(nautobot_wrapper.model, TreeModel):
            for name in ("tree_id", "lft", "rght", "level"):
                self.disable_field(name, "Tree fields doesn't need to be imported")

        self.adapter.logger.debug("Created %s", self)

    def __str__(self) -> str:
        """Return a string representation of the wrapper."""
        return f"{self.__class__.__name__}<{self.content_type} -> {self.nautobot.content_type}>"

    def cache_record_uids(self, source: RecordData, nautobot_uid: Optional[Uid] = None) -> Uid:
        """Cache record identifier mappings.

        When `nautobot_uid` is not provided, it is generated from the source data and caching is processed there.
        """
        if not nautobot_uid:
            return self.get_pk_from_data(source)

        if self.identifiers:
            identifiers_data = [source[field_name] for field_name in self.identifiers]
            self._uid_to_pk_cache[json.dumps(identifiers_data)] = nautobot_uid

        source_uid = source.get(self.nautobot.pk_field.name, None)
        if source_uid and source_uid not in self._uid_to_pk_cache:
            self._uid_to_pk_cache[source_uid] = nautobot_uid

        self._uid_to_pk_cache[nautobot_uid] = nautobot_uid

        return nautobot_uid

    def first_pass(self, data: RecordData) -> None:
        """Firts pass of data import."""
        if self.pre_import:
            if self.pre_import(data, ImporterPass.DEFINE_STRUCTURE) != PreImportResult.USE_RECORD:
                self.stats.first_pass_skipped += 1
                return

        self.stats.first_pass_used += 1

        if self.disable_reason:
            return

        for field_name in data.keys():
            self.add_field(field_name, SourceFieldSource.DATA)

    def second_pass(self, data: RecordData) -> None:
        """Second pass of data import."""
        if self.disable_reason:
            return

        if self.pre_import:
            if self.pre_import(data, ImporterPass.IMPORT_DATA) != PreImportResult.USE_RECORD:
                self.stats.second_pass_skipped += 1
                return

        self.stats.second_pass_used += 1

        self.import_record(data)

    def get_summary(self, content_type_id) -> SourceModelSummary:
        """Get a summary of the model."""
        fields = [field.get_summary() for field in self.fields.values()]

        return SourceModelSummary(
            content_type=self.content_type,
            content_type_id=content_type_id,
            extends_content_type=self.extends_wrapper and self.extends_wrapper.content_type,
            nautobot_content_type=self.nautobot.content_type,
            disable_reason=self.disable_reason,
            identifiers=self.identifiers,
            disable_related_reference=self.disable_related_reference,
            forward_references=self.forward_references and self.forward_references.__name__ or None,
            pre_import=self.pre_import and self.pre_import.__name__ or None,
            fields=sorted(fields, key=lambda field: field.name),
            flags=str(self.flags),
            default_reference_uid=serialize_to_summary(self.default_reference_uid),
            stats=self.stats,
        )

    def set_identifiers(self, identifiers: Iterable[FieldName]) -> None:
        """Set identifiers for the model."""
        if self.identifiers:
            if list(identifiers) == self.identifiers:
                return
            raise ValueError(
                f"Different identifiers were already set up | original: `{self.identifiers}` | new: `{identifiers}`"
            )

        if list(identifiers) == [self.nautobot.pk_field.name]:
            return

        self.identifiers = list(identifiers)
        for identifier in self.identifiers:
            self.add_field(identifier, SourceFieldSource.IDENTIFIER)

    def disable_field(self, field_name: FieldName, reason: str) -> "SourceField":
        """Disable field importing."""
        field = self.add_field(field_name, SourceFieldSource.CUSTOM)
        field.disable(reason)
        return field

    def format_field_name(self, name: FieldName) -> str:
        """Format a field name for logging."""
        return f"{self.content_type}->{name}"

    def add_field(self, name: FieldName, source: SourceFieldSource) -> "SourceField":
        """Add a field definition for a source field."""
        if self.importers is not None:
            raise ValueError(f"Can't add field {self.format_field_name(name)}, model's importers already created.")

        if name not in self.fields:
            return SourceField(self, name, source)

        field = self.fields[name]
        field.sources.add(source)
        return field

    def create_importers(self) -> None:
        """Create importers for all fields."""
        if self.importers is not None:
            raise RuntimeError(f"Importers already created for {self.content_type}")

        if not self.extends_wrapper:
            for field_name in AUTO_ADD_FIELDS:
                if hasattr(self.nautobot.model, field_name):
                    self.add_field(field_name, SourceFieldSource.AUTO)

        while True:
            fields = [field for field in self.fields.values() if not field.processed]
            if not fields:
                break

            for field in fields:
                try:
                    field.create_importer()
                except Exception:
                    self.adapter.logger.error("Failed to create importer for %s", field)
                    raise

        self.importers = set(field.importer for field in self.fields.values() if field.importer)

    def get_pk_from_uid(self, uid: Uid) -> Uid:
        """Get a source primary key for a given source uid."""
        if uid in self._uid_to_pk_cache:
            return self._uid_to_pk_cache[uid]

        if self.nautobot.pk_field.internal_type == InternalFieldType.UUID_FIELD:
            if self.extends_wrapper:
                result = self.extends_wrapper.get_pk_from_uid(uid)
            else:
                result = source_pk_to_uuid(self.content_type or self.content_type, uid)
        elif self.nautobot.pk_field.is_auto_increment:
            self.nautobot.last_id += 1
            result = self.nautobot.last_id
        else:
            raise ValueError(f"Unsupported pk_type {self.nautobot.pk_field.internal_type}")

        self._uid_to_pk_cache[uid] = result
        self._uid_to_pk_cache[result] = result

        return result

    def get_pk_from_identifiers(self, data: Union[Uid, Iterable[Uid]]) -> Uid:
        """Get a source primary key for a given source identifiers."""
        if not self.identifiers:
            if isinstance(data, (UUID, str, int)):
                return self.get_pk_from_uid(data)

            raise ValueError(f"Invalid identifiers {data} for {self.identifiers}")

        if not isinstance(data, list):
            data = list(data)  # type: ignore
        if len(self.identifiers) != len(data):
            raise ValueError(f"Invalid identifiers {data} for {self.identifiers}")

        identifiers_uid = json.dumps(data)
        if identifiers_uid in self._uid_to_pk_cache:
            return self._uid_to_pk_cache[identifiers_uid]

        filter_kwargs = {self.identifiers[index]: value for index, value in enumerate(data)}
        try:
            nautobot_instance = self.nautobot.model.objects.get(**filter_kwargs)
            nautobot_uid = getattr(nautobot_instance, self.nautobot.pk_field.name)
            if not nautobot_uid:
                raise ValueError(f"Invalid args {filter_kwargs} for {nautobot_instance}")
            self._uid_to_pk_cache[identifiers_uid] = nautobot_uid
            self._uid_to_pk_cache[nautobot_uid] = nautobot_uid
            return nautobot_uid
        except self.nautobot.model.DoesNotExist:  # type: ignore
            return self.get_pk_from_uid(identifiers_uid)

    def get_pk_from_data(self, data: RecordData) -> Uid:
        """Get a source primary key for a given source data."""
        if not self.identifiers:
            return self.get_pk_from_uid(data[self.nautobot.pk_field.name])

        data_uid = data.get(self.nautobot.pk_field.name, None)
        if data_uid and data_uid in self._uid_to_pk_cache:
            return self._uid_to_pk_cache[data_uid]

        result = self.get_pk_from_identifiers(data[field_name] for field_name in self.identifiers)

        if data_uid:
            self._uid_to_pk_cache[data_uid] = result

        return result

    def import_record(self, data: RecordData, target: Optional[DiffSyncBaseModel] = None) -> DiffSyncBaseModel:
        """Import a single item from the source."""
        self.adapter.logger.debug("Importing record %s %s", self, data)
        if self.importers is None:
            raise RuntimeError(f"Importers not created for {self}")

        if target:
            uid = getattr(target, self.nautobot.pk_field.name)
        else:
            uid = self.get_pk_from_data(data)
            target = self.get_or_create(uid)

        for importer in self.importers:
            try:
                importer(data, target)
            # pylint: disable=broad-exception-caught
            except Exception as error:
                field = next(field for field in self.fields.values() if field.importer == importer)
                self.nautobot.add_issue(
                    diffsync_instance=target,
                    error=error,
                    message=str({"field": field.name if field else None, "importer": importer.__name__}),
                )

        self.stats.imported += 1
        self.adapter.logger.debug("Imported %s %s", uid, target.get_attrs())

        return target

    def get_or_create(self, uid: Uid, fail_missing=False) -> DiffSyncBaseModel:
        """Get an existing DiffSync Model instance from the source or create a new one.

        Use Nautobot data as defaults if available.
        """
        filter_kwargs = {self.nautobot.pk_field.name: uid}
        diffsync_class = self.nautobot.diffsync_class
        result = self.adapter.get_or_none(diffsync_class, filter_kwargs)
        if result:
            if not isinstance(result, DiffSyncBaseModel):
                raise TypeError(f"Invalid instance type {result}")
            return result

        result = diffsync_class(**filter_kwargs, diffsync=self.adapter)  # type: ignore
        result.model_flags = self.flags

        cached_data = self._cached_data.get(uid, None)
        if cached_data:
            fail_missing = False
            self.import_record(cached_data, result)
            self.stats.imported_from_cache += 1

        nautobot_diffsync_instance = self.nautobot.find_or_create(filter_kwargs)
        if nautobot_diffsync_instance:
            fail_missing = False
            for key, value in nautobot_diffsync_instance.get_attrs().items():
                if value not in EMPTY_VALUES:
                    setattr(result, key, value)

        if fail_missing:
            raise ValueError(f"Missing {self} {uid} in Nautobot or cached data")

        self.adapter.add(result)
        self.stats.created += 1
        if self.flags == DiffSyncModelFlags.IGNORE:
            self.nautobot.stats.source_ignored += 1
        else:
            self.nautobot.stats.source_created += 1

        return result

    def get_default_reference_uid(self) -> Uid:
        """Get the default reference to this model."""
        if self.default_reference_uid:
            return self.default_reference_uid
        raise ValueError("Missing default reference")

    def cache_record(self, data: RecordData) -> Uid:
        """Cache data for optional later use.

        If record is referenced by other models, it will be imported automatically; otherwise, it will be ignored.
        """
        uid = self.get_pk_from_data(data)
        if uid in self._cached_data:
            return uid

        if self.importers is None:
            for field_name in data.keys():
                self.add_field(field_name, SourceFieldSource.CACHE)

        self._cached_data[uid] = data
        self.stats.pre_cached += 1

        self.adapter.logger.debug("Cached %s %s %s", self, uid, data)

        return uid

    def set_default_reference(self, data: RecordData) -> None:
        """Set the default reference to this model."""
        self.default_reference_uid = self.cache_record(data)

    def post_import(self) -> bool:
        """Post import processing.

        Assigns referenced content_types to referencing instances.

        Returns False if no post processing is needed, otherwise True to indicate that post processing is needed.
        """
        if not self.references:
            return False

        references = self.references
        self.references = {}

        if self.forward_references:
            self.forward_references(self, references)
            return True

        for uid, content_types in references.items():
            # Keep this even when no content_types field is present, to create referenced cached data
            instance = self.get_or_create(uid, fail_missing=True)
            if "content_types" not in self.nautobot.fields:
                continue

            content_types = set(wrapper.nautobot.content_type_instance.pk for wrapper in content_types)
            target_content_types = getattr(instance, "content_types", None)
            if target_content_types != content_types:
                if target_content_types:
                    target_content_types.update(content_types)
                else:
                    instance.content_types = content_types
                self.adapter.update(instance)

        return True

    def add_reference(self, related_wrapper: "SourceModelWrapper", uid: Uid) -> None:
        """Add a reference from this content type to related record."""
        if self.disable_related_reference:
            return
        self.adapter.logger.debug(
            "Adding reference from: %s to: %s %s", self.content_type, related_wrapper.content_type, uid
        )
        if not uid:
            raise ValueError(f"Invalid uid {uid}")
        related_wrapper.references.setdefault(uid, set()).add(self)
__init__(adapter, content_type, nautobot_wrapper)

Initialize the SourceModelWrapper.

Source code in nautobot_netbox_importer/generator/source.py
def __init__(self, adapter: SourceAdapter, content_type: ContentTypeStr, nautobot_wrapper: NautobotModelWrapper):
    """Initialize the SourceModelWrapper."""
    if content_type in adapter.wrappers:
        raise ValueError(f"Duplicate content type {content_type}")
    adapter.wrappers[content_type] = self
    self.adapter = adapter
    self.content_type = content_type
    self.nautobot = nautobot_wrapper
    if self.nautobot.disabled:
        self.disable_reason = f"Nautobot content type: `{nautobot_wrapper.content_type}` not found"
    else:
        self.disable_reason = ""

    # Source field names when referencing this model
    self.identifiers: Optional[List[FieldName]] = None

    # Used to autofill `content_types` field
    self.disable_related_reference = False
    self.references: SourceReferences = {}
    self.forward_references: Optional[ForwardReferences] = None

    # Whether importing record data exteds existing record
    self.extends_wrapper: Optional[SourceModelWrapper] = None

    # Importers are created after all fields are defined
    self.importers: Optional[Set[SourceFieldImporter]] = None

    # Default reference to this model
    self.default_reference_uid: Optional[Uid] = None

    # Caching
    self._uid_to_pk_cache: Dict[Uid, Uid] = {}
    self._cached_data: Dict[Uid, RecordData] = {}

    self.stats = SourceModelStats()
    self.flags = DiffSyncModelFlags.NONE

    # Source fields defintions
    self.fields: OrderedDict[FieldName, SourceField] = OrderedDict()
    self.pre_import: Optional[PreImport] = None

    if self.disable_reason:
        self.adapter.logger.debug("Created disabled %s", self)
        return

    pk_field = self.add_field(nautobot_wrapper.pk_field.name, SourceFieldSource.AUTO)
    pk_field.set_nautobot_field()
    pk_field.processed = True

    if issubclass(nautobot_wrapper.model, TreeModel):
        for name in ("tree_id", "lft", "rght", "level"):
            self.disable_field(name, "Tree fields doesn't need to be imported")

    self.adapter.logger.debug("Created %s", self)
__str__()

Return a string representation of the wrapper.

Source code in nautobot_netbox_importer/generator/source.py
def __str__(self) -> str:
    """Return a string representation of the wrapper."""
    return f"{self.__class__.__name__}<{self.content_type} -> {self.nautobot.content_type}>"
add_field(name, source)

Add a field definition for a source field.

Source code in nautobot_netbox_importer/generator/source.py
def add_field(self, name: FieldName, source: SourceFieldSource) -> "SourceField":
    """Add a field definition for a source field."""
    if self.importers is not None:
        raise ValueError(f"Can't add field {self.format_field_name(name)}, model's importers already created.")

    if name not in self.fields:
        return SourceField(self, name, source)

    field = self.fields[name]
    field.sources.add(source)
    return field
add_reference(related_wrapper, uid)

Add a reference from this content type to related record.

Source code in nautobot_netbox_importer/generator/source.py
def add_reference(self, related_wrapper: "SourceModelWrapper", uid: Uid) -> None:
    """Add a reference from this content type to related record."""
    if self.disable_related_reference:
        return
    self.adapter.logger.debug(
        "Adding reference from: %s to: %s %s", self.content_type, related_wrapper.content_type, uid
    )
    if not uid:
        raise ValueError(f"Invalid uid {uid}")
    related_wrapper.references.setdefault(uid, set()).add(self)
cache_record(data)

Cache data for optional later use.

If record is referenced by other models, it will be imported automatically; otherwise, it will be ignored.

Source code in nautobot_netbox_importer/generator/source.py
def cache_record(self, data: RecordData) -> Uid:
    """Cache data for optional later use.

    If record is referenced by other models, it will be imported automatically; otherwise, it will be ignored.
    """
    uid = self.get_pk_from_data(data)
    if uid in self._cached_data:
        return uid

    if self.importers is None:
        for field_name in data.keys():
            self.add_field(field_name, SourceFieldSource.CACHE)

    self._cached_data[uid] = data
    self.stats.pre_cached += 1

    self.adapter.logger.debug("Cached %s %s %s", self, uid, data)

    return uid
cache_record_uids(source, nautobot_uid=None)

Cache record identifier mappings.

When nautobot_uid is not provided, it is generated from the source data and caching is processed there.

Source code in nautobot_netbox_importer/generator/source.py
def cache_record_uids(self, source: RecordData, nautobot_uid: Optional[Uid] = None) -> Uid:
    """Cache record identifier mappings.

    When `nautobot_uid` is not provided, it is generated from the source data and caching is processed there.
    """
    if not nautobot_uid:
        return self.get_pk_from_data(source)

    if self.identifiers:
        identifiers_data = [source[field_name] for field_name in self.identifiers]
        self._uid_to_pk_cache[json.dumps(identifiers_data)] = nautobot_uid

    source_uid = source.get(self.nautobot.pk_field.name, None)
    if source_uid and source_uid not in self._uid_to_pk_cache:
        self._uid_to_pk_cache[source_uid] = nautobot_uid

    self._uid_to_pk_cache[nautobot_uid] = nautobot_uid

    return nautobot_uid
create_importers()

Create importers for all fields.

Source code in nautobot_netbox_importer/generator/source.py
def create_importers(self) -> None:
    """Create importers for all fields."""
    if self.importers is not None:
        raise RuntimeError(f"Importers already created for {self.content_type}")

    if not self.extends_wrapper:
        for field_name in AUTO_ADD_FIELDS:
            if hasattr(self.nautobot.model, field_name):
                self.add_field(field_name, SourceFieldSource.AUTO)

    while True:
        fields = [field for field in self.fields.values() if not field.processed]
        if not fields:
            break

        for field in fields:
            try:
                field.create_importer()
            except Exception:
                self.adapter.logger.error("Failed to create importer for %s", field)
                raise

    self.importers = set(field.importer for field in self.fields.values() if field.importer)
disable_field(field_name, reason)

Disable field importing.

Source code in nautobot_netbox_importer/generator/source.py
def disable_field(self, field_name: FieldName, reason: str) -> "SourceField":
    """Disable field importing."""
    field = self.add_field(field_name, SourceFieldSource.CUSTOM)
    field.disable(reason)
    return field
first_pass(data)

Firts pass of data import.

Source code in nautobot_netbox_importer/generator/source.py
def first_pass(self, data: RecordData) -> None:
    """Firts pass of data import."""
    if self.pre_import:
        if self.pre_import(data, ImporterPass.DEFINE_STRUCTURE) != PreImportResult.USE_RECORD:
            self.stats.first_pass_skipped += 1
            return

    self.stats.first_pass_used += 1

    if self.disable_reason:
        return

    for field_name in data.keys():
        self.add_field(field_name, SourceFieldSource.DATA)
format_field_name(name)

Format a field name for logging.

Source code in nautobot_netbox_importer/generator/source.py
def format_field_name(self, name: FieldName) -> str:
    """Format a field name for logging."""
    return f"{self.content_type}->{name}"
get_default_reference_uid()

Get the default reference to this model.

Source code in nautobot_netbox_importer/generator/source.py
def get_default_reference_uid(self) -> Uid:
    """Get the default reference to this model."""
    if self.default_reference_uid:
        return self.default_reference_uid
    raise ValueError("Missing default reference")
get_or_create(uid, fail_missing=False)

Get an existing DiffSync Model instance from the source or create a new one.

Use Nautobot data as defaults if available.

Source code in nautobot_netbox_importer/generator/source.py
def get_or_create(self, uid: Uid, fail_missing=False) -> DiffSyncBaseModel:
    """Get an existing DiffSync Model instance from the source or create a new one.

    Use Nautobot data as defaults if available.
    """
    filter_kwargs = {self.nautobot.pk_field.name: uid}
    diffsync_class = self.nautobot.diffsync_class
    result = self.adapter.get_or_none(diffsync_class, filter_kwargs)
    if result:
        if not isinstance(result, DiffSyncBaseModel):
            raise TypeError(f"Invalid instance type {result}")
        return result

    result = diffsync_class(**filter_kwargs, diffsync=self.adapter)  # type: ignore
    result.model_flags = self.flags

    cached_data = self._cached_data.get(uid, None)
    if cached_data:
        fail_missing = False
        self.import_record(cached_data, result)
        self.stats.imported_from_cache += 1

    nautobot_diffsync_instance = self.nautobot.find_or_create(filter_kwargs)
    if nautobot_diffsync_instance:
        fail_missing = False
        for key, value in nautobot_diffsync_instance.get_attrs().items():
            if value not in EMPTY_VALUES:
                setattr(result, key, value)

    if fail_missing:
        raise ValueError(f"Missing {self} {uid} in Nautobot or cached data")

    self.adapter.add(result)
    self.stats.created += 1
    if self.flags == DiffSyncModelFlags.IGNORE:
        self.nautobot.stats.source_ignored += 1
    else:
        self.nautobot.stats.source_created += 1

    return result
get_pk_from_data(data)

Get a source primary key for a given source data.

Source code in nautobot_netbox_importer/generator/source.py
def get_pk_from_data(self, data: RecordData) -> Uid:
    """Get a source primary key for a given source data."""
    if not self.identifiers:
        return self.get_pk_from_uid(data[self.nautobot.pk_field.name])

    data_uid = data.get(self.nautobot.pk_field.name, None)
    if data_uid and data_uid in self._uid_to_pk_cache:
        return self._uid_to_pk_cache[data_uid]

    result = self.get_pk_from_identifiers(data[field_name] for field_name in self.identifiers)

    if data_uid:
        self._uid_to_pk_cache[data_uid] = result

    return result
get_pk_from_identifiers(data)

Get a source primary key for a given source identifiers.

Source code in nautobot_netbox_importer/generator/source.py
def get_pk_from_identifiers(self, data: Union[Uid, Iterable[Uid]]) -> Uid:
    """Get a source primary key for a given source identifiers."""
    if not self.identifiers:
        if isinstance(data, (UUID, str, int)):
            return self.get_pk_from_uid(data)

        raise ValueError(f"Invalid identifiers {data} for {self.identifiers}")

    if not isinstance(data, list):
        data = list(data)  # type: ignore
    if len(self.identifiers) != len(data):
        raise ValueError(f"Invalid identifiers {data} for {self.identifiers}")

    identifiers_uid = json.dumps(data)
    if identifiers_uid in self._uid_to_pk_cache:
        return self._uid_to_pk_cache[identifiers_uid]

    filter_kwargs = {self.identifiers[index]: value for index, value in enumerate(data)}
    try:
        nautobot_instance = self.nautobot.model.objects.get(**filter_kwargs)
        nautobot_uid = getattr(nautobot_instance, self.nautobot.pk_field.name)
        if not nautobot_uid:
            raise ValueError(f"Invalid args {filter_kwargs} for {nautobot_instance}")
        self._uid_to_pk_cache[identifiers_uid] = nautobot_uid
        self._uid_to_pk_cache[nautobot_uid] = nautobot_uid
        return nautobot_uid
    except self.nautobot.model.DoesNotExist:  # type: ignore
        return self.get_pk_from_uid(identifiers_uid)
get_pk_from_uid(uid)

Get a source primary key for a given source uid.

Source code in nautobot_netbox_importer/generator/source.py
def get_pk_from_uid(self, uid: Uid) -> Uid:
    """Get a source primary key for a given source uid."""
    if uid in self._uid_to_pk_cache:
        return self._uid_to_pk_cache[uid]

    if self.nautobot.pk_field.internal_type == InternalFieldType.UUID_FIELD:
        if self.extends_wrapper:
            result = self.extends_wrapper.get_pk_from_uid(uid)
        else:
            result = source_pk_to_uuid(self.content_type or self.content_type, uid)
    elif self.nautobot.pk_field.is_auto_increment:
        self.nautobot.last_id += 1
        result = self.nautobot.last_id
    else:
        raise ValueError(f"Unsupported pk_type {self.nautobot.pk_field.internal_type}")

    self._uid_to_pk_cache[uid] = result
    self._uid_to_pk_cache[result] = result

    return result
get_summary(content_type_id)

Get a summary of the model.

Source code in nautobot_netbox_importer/generator/source.py
def get_summary(self, content_type_id) -> SourceModelSummary:
    """Get a summary of the model."""
    fields = [field.get_summary() for field in self.fields.values()]

    return SourceModelSummary(
        content_type=self.content_type,
        content_type_id=content_type_id,
        extends_content_type=self.extends_wrapper and self.extends_wrapper.content_type,
        nautobot_content_type=self.nautobot.content_type,
        disable_reason=self.disable_reason,
        identifiers=self.identifiers,
        disable_related_reference=self.disable_related_reference,
        forward_references=self.forward_references and self.forward_references.__name__ or None,
        pre_import=self.pre_import and self.pre_import.__name__ or None,
        fields=sorted(fields, key=lambda field: field.name),
        flags=str(self.flags),
        default_reference_uid=serialize_to_summary(self.default_reference_uid),
        stats=self.stats,
    )
import_record(data, target=None)

Import a single item from the source.

Source code in nautobot_netbox_importer/generator/source.py
def import_record(self, data: RecordData, target: Optional[DiffSyncBaseModel] = None) -> DiffSyncBaseModel:
    """Import a single item from the source."""
    self.adapter.logger.debug("Importing record %s %s", self, data)
    if self.importers is None:
        raise RuntimeError(f"Importers not created for {self}")

    if target:
        uid = getattr(target, self.nautobot.pk_field.name)
    else:
        uid = self.get_pk_from_data(data)
        target = self.get_or_create(uid)

    for importer in self.importers:
        try:
            importer(data, target)
        # pylint: disable=broad-exception-caught
        except Exception as error:
            field = next(field for field in self.fields.values() if field.importer == importer)
            self.nautobot.add_issue(
                diffsync_instance=target,
                error=error,
                message=str({"field": field.name if field else None, "importer": importer.__name__}),
            )

    self.stats.imported += 1
    self.adapter.logger.debug("Imported %s %s", uid, target.get_attrs())

    return target
post_import()

Post import processing.

Assigns referenced content_types to referencing instances.

Returns False if no post processing is needed, otherwise True to indicate that post processing is needed.

Source code in nautobot_netbox_importer/generator/source.py
def post_import(self) -> bool:
    """Post import processing.

    Assigns referenced content_types to referencing instances.

    Returns False if no post processing is needed, otherwise True to indicate that post processing is needed.
    """
    if not self.references:
        return False

    references = self.references
    self.references = {}

    if self.forward_references:
        self.forward_references(self, references)
        return True

    for uid, content_types in references.items():
        # Keep this even when no content_types field is present, to create referenced cached data
        instance = self.get_or_create(uid, fail_missing=True)
        if "content_types" not in self.nautobot.fields:
            continue

        content_types = set(wrapper.nautobot.content_type_instance.pk for wrapper in content_types)
        target_content_types = getattr(instance, "content_types", None)
        if target_content_types != content_types:
            if target_content_types:
                target_content_types.update(content_types)
            else:
                instance.content_types = content_types
            self.adapter.update(instance)

    return True
second_pass(data)

Second pass of data import.

Source code in nautobot_netbox_importer/generator/source.py
def second_pass(self, data: RecordData) -> None:
    """Second pass of data import."""
    if self.disable_reason:
        return

    if self.pre_import:
        if self.pre_import(data, ImporterPass.IMPORT_DATA) != PreImportResult.USE_RECORD:
            self.stats.second_pass_skipped += 1
            return

    self.stats.second_pass_used += 1

    self.import_record(data)
set_default_reference(data)

Set the default reference to this model.

Source code in nautobot_netbox_importer/generator/source.py
def set_default_reference(self, data: RecordData) -> None:
    """Set the default reference to this model."""
    self.default_reference_uid = self.cache_record(data)
set_identifiers(identifiers)

Set identifiers for the model.

Source code in nautobot_netbox_importer/generator/source.py
def set_identifiers(self, identifiers: Iterable[FieldName]) -> None:
    """Set identifiers for the model."""
    if self.identifiers:
        if list(identifiers) == self.identifiers:
            return
        raise ValueError(
            f"Different identifiers were already set up | original: `{self.identifiers}` | new: `{identifiers}`"
        )

    if list(identifiers) == [self.nautobot.pk_field.name]:
        return

    self.identifiers = list(identifiers)
    for identifier in self.identifiers:
        self.add_field(identifier, SourceFieldSource.IDENTIFIER)
SourceRecord

Bases: NamedTuple

Source Data Item.

Source code in nautobot_netbox_importer/generator/source.py
class SourceRecord(NamedTuple):
    """Source Data Item."""

    content_type: ContentTypeStr
    data: RecordData

summary

Importer summary module.

FieldSummary

Bases: NamedTuple

Field summary.

Source code in nautobot_netbox_importer/summary.py
class FieldSummary(NamedTuple):
    """Field summary."""

    name: FieldName
    nautobot_name: Optional[FieldName]
    nautobot_internal_type: Optional[str]
    nautobot_can_import: Optional[bool]
    importer: Optional[str]
    definition: Union[str, bool, int, float, None]
    sources: List[str]
    default_value: Union[str, bool, int, float, None]
    disable_reason: str
    required: bool

ImportSummary

Import summary.

Source code in nautobot_netbox_importer/summary.py
class ImportSummary:
    """Import summary."""

    def __init__(self):
        """Initialize the import summary."""
        self.source: List[SourceModelSummary] = []
        self.nautobot: List[NautobotModelSummary] = []
        self.diffsync: DiffSyncSummary = {}

    @property
    def data_sources(self) -> Generator[SourceModelSummary, None, None]:
        """Get source originating from data."""
        for summary in self.source:
            if any(field for field in summary.fields if "DATA" in field.sources):
                yield summary

    @property
    def data_nautobot_models(self) -> Generator[NautobotModelSummary, None, None]:
        """Get Nautobot models originating from data."""
        content_types = set(item.content_type for item in self.data_sources)

        for item in self.nautobot:
            if item.content_type in content_types:
                yield item

    def load(self, path: Pathable):
        """Load the summary from a file."""
        content = json.loads(Path(path).read_text(encoding="utf-8"))

        self.diffsync = content["diffsync"]

        for model in content["source"].values():
            stats = SourceModelStats()
            for key, value in model.pop("stats", {}).items():
                setattr(stats, key, value)
            fields = model.pop("fields", [])
            self.source.append(
                SourceModelSummary(
                    **model,
                    fields=[FieldSummary(**field) for field in fields.values()],
                    stats=stats,
                )
            )

        for model in content["nautobot"].values():
            stats = NautobotModelStats()
            for key, value in model.pop("stats", {}).items():
                setattr(stats, key, value)
            issues = model.pop("issues", {})
            self.nautobot.append(
                NautobotModelSummary(
                    **model,
                    stats=stats,
                    issues=[ImporterIssue(**issue) for issue in issues],
                )
            )

    def dump(self, path: Pathable, output_format="json", indent=4):
        """Dump the summary to a file."""
        if output_format == "json":
            Path(path).write_text(
                json.dumps(
                    {
                        "diffsync": self.diffsync,
                        "source": {
                            summary.content_type: {
                                **summary._asdict(),
                                "fields": {field.name: field._asdict() for field in summary.fields},
                                "stats": summary.stats.__dict__,
                            }
                            for summary in self.source
                        },
                        "nautobot": {
                            summary.content_type: {
                                **summary._asdict(),
                                "issues": [issue._asdict() for issue in summary.issues],
                                "stats": summary.stats.__dict__,
                            }
                            for summary in self.nautobot
                        },
                    },
                    indent=indent,
                ),
                encoding="utf-8",
            )
        elif output_format == "text":
            with open(path, "w", encoding="utf-8") as file:
                for line in self.get_summary():
                    file.write(line + "\n")
        else:
            raise ValueError(f"Unsupported format {output_format}")

    def print(self):
        """Print a summary of the import."""
        for line in self.get_summary():
            print(line)

    def get_summary(self) -> Generator[str, None, None]:
        """Get a summary of the import."""
        yield _fill_up("* Import Summary:")

        yield _fill_up("= DiffSync Summary:")
        for key, value in self.diffsync.items():
            yield f"{key}: {value}"

        yield from self.get_stats("Source", self.source)
        yield from self.get_stats("Nautobot", self.nautobot)
        yield from self.get_content_types_deviations()
        yield from self.get_back_mapping()
        yield from self.get_issues()
        yield from self.get_fields_mapping()

        yield _fill_up("* End of Import Summary")

    def get_stats(self, caption: str, objects: Iterable[object]) -> Generator[str, None, None]:
        """Get formatted stats."""
        yield _fill_up("=", caption, "Stats:")
        for summary in objects:
            stats = getattr(summary, "stats", {}).__dict__
            if stats:
                yield _fill_up("-", getattr(summary, "content_type"))
                for key, value in stats.items():
                    yield f"{key}: {value}"

    def get_content_types_deviations(self) -> Generator[str, None, None]:
        """Get formatted content types deviations."""
        yield _fill_up("= Content Types Mapping Deviations:")
        yield "  Mapping deviations from source content type to Nautobot content type"

        for summary in self.data_sources:
            if summary.disable_reason:
                yield f"{summary.content_type} => {summary.nautobot_content_type} | Disabled with reason: {summary.disable_reason}"
            elif summary.extends_content_type:
                yield f"{summary.content_type} EXTENDS {summary.extends_content_type} => {summary.nautobot_content_type}"
            elif summary.content_type != summary.nautobot_content_type:
                yield f"{summary.content_type} => {summary.nautobot_content_type}"

    def get_back_mapping(self) -> Generator[str, None, None]:
        """Get formatted back mapping."""
        yield _fill_up("= Content Types Back Mapping:")
        yield "  Back mapping deviations from Nautobot content type to the source content type"

        back_mapping = {}

        for summary in self.data_sources:
            if summary.nautobot_content_type != summary.content_type:
                if summary.nautobot_content_type in back_mapping:
                    if back_mapping[summary.nautobot_content_type] != summary.content_type:
                        back_mapping[summary.nautobot_content_type] = None
                else:
                    back_mapping[summary.nautobot_content_type] = summary.content_type

        for nautobot_content_type, content_type in back_mapping.items():
            if content_type:
                yield f"{nautobot_content_type} => {content_type}"
            else:
                yield f"{nautobot_content_type} => Ambiguous"

    def get_issues(self) -> Generator[str, None, None]:
        """Get formatted issues."""
        yield _fill_up("= Importer issues:")
        for summary in self.nautobot:
            if summary.issues:
                yield _fill_up("-", summary.content_type)
                for issue in summary.issues:
                    yield f"{issue.uid} | {issue.issue_type} | {json.dumps(issue.name)} | {json.dumps(issue.message)}"

    def get_fields_mapping(self) -> Generator[str, None, None]:
        """Get formatted field mappings."""
        yield _fill_up("= Field Mappings:")

        def get_field(field: FieldSummary):
            yield field.name
            yield "=>"

            if field.disable_reason:
                yield "Disabled with reason:"
                yield field.disable_reason
                return

            if field.importer:
                yield field.importer
            elif field.name == "id":
                yield "uid_from_data"
            else:
                yield "NO IMPORTER"

            yield "=>"

            if field.nautobot_name:
                yield field.nautobot_name
                yield f"({field.nautobot_internal_type})"
            else:
                yield "CUSTOM TARGET"

        for summary in self.data_sources:
            yield _fill_up(
                "-",
                summary.content_type,
                "=>",
                summary.nautobot_content_type,
            )

            if summary.disable_reason:
                yield f"    Disable reason: {summary.disable_reason}"
            else:
                for field in summary.fields:
                    if "DATA" in field.sources:
                        yield " ".join(get_field(field))
data_nautobot_models: Generator[NautobotModelSummary, None, None] property

Get Nautobot models originating from data.

data_sources: Generator[SourceModelSummary, None, None] property

Get source originating from data.

__init__()

Initialize the import summary.

Source code in nautobot_netbox_importer/summary.py
def __init__(self):
    """Initialize the import summary."""
    self.source: List[SourceModelSummary] = []
    self.nautobot: List[NautobotModelSummary] = []
    self.diffsync: DiffSyncSummary = {}
dump(path, output_format='json', indent=4)

Dump the summary to a file.

Source code in nautobot_netbox_importer/summary.py
def dump(self, path: Pathable, output_format="json", indent=4):
    """Dump the summary to a file."""
    if output_format == "json":
        Path(path).write_text(
            json.dumps(
                {
                    "diffsync": self.diffsync,
                    "source": {
                        summary.content_type: {
                            **summary._asdict(),
                            "fields": {field.name: field._asdict() for field in summary.fields},
                            "stats": summary.stats.__dict__,
                        }
                        for summary in self.source
                    },
                    "nautobot": {
                        summary.content_type: {
                            **summary._asdict(),
                            "issues": [issue._asdict() for issue in summary.issues],
                            "stats": summary.stats.__dict__,
                        }
                        for summary in self.nautobot
                    },
                },
                indent=indent,
            ),
            encoding="utf-8",
        )
    elif output_format == "text":
        with open(path, "w", encoding="utf-8") as file:
            for line in self.get_summary():
                file.write(line + "\n")
    else:
        raise ValueError(f"Unsupported format {output_format}")
get_back_mapping()

Get formatted back mapping.

Source code in nautobot_netbox_importer/summary.py
def get_back_mapping(self) -> Generator[str, None, None]:
    """Get formatted back mapping."""
    yield _fill_up("= Content Types Back Mapping:")
    yield "  Back mapping deviations from Nautobot content type to the source content type"

    back_mapping = {}

    for summary in self.data_sources:
        if summary.nautobot_content_type != summary.content_type:
            if summary.nautobot_content_type in back_mapping:
                if back_mapping[summary.nautobot_content_type] != summary.content_type:
                    back_mapping[summary.nautobot_content_type] = None
            else:
                back_mapping[summary.nautobot_content_type] = summary.content_type

    for nautobot_content_type, content_type in back_mapping.items():
        if content_type:
            yield f"{nautobot_content_type} => {content_type}"
        else:
            yield f"{nautobot_content_type} => Ambiguous"
get_content_types_deviations()

Get formatted content types deviations.

Source code in nautobot_netbox_importer/summary.py
def get_content_types_deviations(self) -> Generator[str, None, None]:
    """Get formatted content types deviations."""
    yield _fill_up("= Content Types Mapping Deviations:")
    yield "  Mapping deviations from source content type to Nautobot content type"

    for summary in self.data_sources:
        if summary.disable_reason:
            yield f"{summary.content_type} => {summary.nautobot_content_type} | Disabled with reason: {summary.disable_reason}"
        elif summary.extends_content_type:
            yield f"{summary.content_type} EXTENDS {summary.extends_content_type} => {summary.nautobot_content_type}"
        elif summary.content_type != summary.nautobot_content_type:
            yield f"{summary.content_type} => {summary.nautobot_content_type}"
get_fields_mapping()

Get formatted field mappings.

Source code in nautobot_netbox_importer/summary.py
def get_fields_mapping(self) -> Generator[str, None, None]:
    """Get formatted field mappings."""
    yield _fill_up("= Field Mappings:")

    def get_field(field: FieldSummary):
        yield field.name
        yield "=>"

        if field.disable_reason:
            yield "Disabled with reason:"
            yield field.disable_reason
            return

        if field.importer:
            yield field.importer
        elif field.name == "id":
            yield "uid_from_data"
        else:
            yield "NO IMPORTER"

        yield "=>"

        if field.nautobot_name:
            yield field.nautobot_name
            yield f"({field.nautobot_internal_type})"
        else:
            yield "CUSTOM TARGET"

    for summary in self.data_sources:
        yield _fill_up(
            "-",
            summary.content_type,
            "=>",
            summary.nautobot_content_type,
        )

        if summary.disable_reason:
            yield f"    Disable reason: {summary.disable_reason}"
        else:
            for field in summary.fields:
                if "DATA" in field.sources:
                    yield " ".join(get_field(field))
get_issues()

Get formatted issues.

Source code in nautobot_netbox_importer/summary.py
def get_issues(self) -> Generator[str, None, None]:
    """Get formatted issues."""
    yield _fill_up("= Importer issues:")
    for summary in self.nautobot:
        if summary.issues:
            yield _fill_up("-", summary.content_type)
            for issue in summary.issues:
                yield f"{issue.uid} | {issue.issue_type} | {json.dumps(issue.name)} | {json.dumps(issue.message)}"
get_stats(caption, objects)

Get formatted stats.

Source code in nautobot_netbox_importer/summary.py
def get_stats(self, caption: str, objects: Iterable[object]) -> Generator[str, None, None]:
    """Get formatted stats."""
    yield _fill_up("=", caption, "Stats:")
    for summary in objects:
        stats = getattr(summary, "stats", {}).__dict__
        if stats:
            yield _fill_up("-", getattr(summary, "content_type"))
            for key, value in stats.items():
                yield f"{key}: {value}"
get_summary()

Get a summary of the import.

Source code in nautobot_netbox_importer/summary.py
def get_summary(self) -> Generator[str, None, None]:
    """Get a summary of the import."""
    yield _fill_up("* Import Summary:")

    yield _fill_up("= DiffSync Summary:")
    for key, value in self.diffsync.items():
        yield f"{key}: {value}"

    yield from self.get_stats("Source", self.source)
    yield from self.get_stats("Nautobot", self.nautobot)
    yield from self.get_content_types_deviations()
    yield from self.get_back_mapping()
    yield from self.get_issues()
    yield from self.get_fields_mapping()

    yield _fill_up("* End of Import Summary")
load(path)

Load the summary from a file.

Source code in nautobot_netbox_importer/summary.py
def load(self, path: Pathable):
    """Load the summary from a file."""
    content = json.loads(Path(path).read_text(encoding="utf-8"))

    self.diffsync = content["diffsync"]

    for model in content["source"].values():
        stats = SourceModelStats()
        for key, value in model.pop("stats", {}).items():
            setattr(stats, key, value)
        fields = model.pop("fields", [])
        self.source.append(
            SourceModelSummary(
                **model,
                fields=[FieldSummary(**field) for field in fields.values()],
                stats=stats,
            )
        )

    for model in content["nautobot"].values():
        stats = NautobotModelStats()
        for key, value in model.pop("stats", {}).items():
            setattr(stats, key, value)
        issues = model.pop("issues", {})
        self.nautobot.append(
            NautobotModelSummary(
                **model,
                stats=stats,
                issues=[ImporterIssue(**issue) for issue in issues],
            )
        )
print()

Print a summary of the import.

Source code in nautobot_netbox_importer/summary.py
def print(self):
    """Print a summary of the import."""
    for line in self.get_summary():
        print(line)

ImporterIssue

Bases: NamedTuple

Importer issue.

Source code in nautobot_netbox_importer/summary.py
class ImporterIssue(NamedTuple):
    """Importer issue."""

    uid: str
    name: str
    issue_type: str
    message: str
    data: Dict[str, str]

NautobotModelStats

Nautobot Model Statistics.

Source code in nautobot_netbox_importer/summary.py
class NautobotModelStats:
    """Nautobot Model Statistics."""

    # Source DiffSyncModels created but ignored by DiffSync
    source_ignored = 0
    # Source DiffSyncModels created and synced by DiffSync
    source_created = 0
    issues = 0
    # Number of Nautobot instances that failed `save()` method
    save_failed = 0
    created = 0
    updated = 0

NautobotModelSummary

Bases: NamedTuple

Nautobot Model Summary.

Source code in nautobot_netbox_importer/summary.py
class NautobotModelSummary(NamedTuple):
    """Nautobot Model Summary."""

    content_type: ContentTypeStr
    content_type_id: Optional[int]
    flags: str
    disabled: bool
    stats: NautobotModelStats
    issues: List[ImporterIssue]

SourceModelStats

Source Model Statistics.

Source code in nautobot_netbox_importer/summary.py
class SourceModelStats:
    """Source Model Statistics."""

    first_pass_skipped = 0
    first_pass_used = 0
    second_pass_skipped = 0
    second_pass_used = 0
    pre_cached = 0
    imported_from_cache = 0
    # Imported using `wrapper.import_data()` including cache and custom importers
    imported = 0
    # DiffSyncModels created
    created = 0

SourceModelSummary

Bases: NamedTuple

Source Model Summary.

Source code in nautobot_netbox_importer/summary.py
class SourceModelSummary(NamedTuple):
    """Source Model Summary."""

    content_type: ContentTypeStr
    content_type_id: int
    extends_content_type: Optional[ContentTypeStr]
    nautobot_content_type: ContentTypeStr
    disable_reason: str
    identifiers: Optional[List[FieldName]]
    disable_related_reference: bool
    forward_references: Optional[str]
    pre_import: Optional[str]
    fields: List[FieldSummary]
    flags: str
    default_reference_uid: Union[str, bool, int, float, None]
    stats: SourceModelStats

serialize_to_summary(value)

Serialize value to summary.

Source code in nautobot_netbox_importer/summary.py
def serialize_to_summary(value):
    """Serialize value to summary."""
    if value is None:
        return None
    if isinstance(value, (str, bool, int, float)):
        return value
    if isinstance(value, Callable):
        return value.__name__
    return str(value)

tests

Unit tests for nautobot_netbox_importer app.

test_basic

Basic tests that do not require Django.

TestDocsPackaging

Bases: TestCase

Test Version in doc requirements is the same pyproject.

Source code in nautobot_netbox_importer/tests/test_basic.py
class TestDocsPackaging(unittest.TestCase):
    """Test Version in doc requirements is the same pyproject."""

    def test_version(self):
        """Verify that pyproject.toml dev dependencies have the same versions as in the docs requirements.txt."""
        parent_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
        poetry_path = os.path.join(parent_path, "pyproject.toml")
        poetry_details = toml.load(poetry_path)["tool"]["poetry"]["group"]["dev"]["dependencies"]
        with open(f"{parent_path}/docs/requirements.txt", "r", encoding="utf-8") as file:
            requirements = [line for line in file.read().splitlines() if (len(line) > 0 and not line.startswith("#"))]
        for pkg in requirements:
            package_name = pkg
            if len(pkg.split("==")) == 2:  # noqa: PLR2004
                package_name, version = pkg.split("==")
            else:
                version = "*"
            self.assertEqual(poetry_details[package_name], version)
test_version()

Verify that pyproject.toml dev dependencies have the same versions as in the docs requirements.txt.

Source code in nautobot_netbox_importer/tests/test_basic.py
def test_version(self):
    """Verify that pyproject.toml dev dependencies have the same versions as in the docs requirements.txt."""
    parent_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
    poetry_path = os.path.join(parent_path, "pyproject.toml")
    poetry_details = toml.load(poetry_path)["tool"]["poetry"]["group"]["dev"]["dependencies"]
    with open(f"{parent_path}/docs/requirements.txt", "r", encoding="utf-8") as file:
        requirements = [line for line in file.read().splitlines() if (len(line) > 0 and not line.startswith("#"))]
    for pkg in requirements:
        package_name = pkg
        if len(pkg.split("==")) == 2:  # noqa: PLR2004
            package_name, version = pkg.split("==")
        else:
            version = "*"
        self.assertEqual(poetry_details[package_name], version)

test_import

Test cases for the NetBox adapter.

Tests use stored fixtures to verify that the import process works as expected.

Check the fixtures README for more details.

TestImport

Bases: TestCase

Unittest for NetBox adapter.

Test cases are dynamically created based on the fixtures available for the current Nautobot version.

Source code in nautobot_netbox_importer/tests/test_import.py
@patch("django.conf.settings.SECRET_KEY", "testing_secret_key")
class TestImport(TestCase):
    """Unittest for NetBox adapter.

    Test cases are dynamically created based on the fixtures available for the current Nautobot version.
    """

    def setUp(self):
        """Set up test environment."""
        super().setUp()

        apps.get_model("extras", "Role").objects.all().delete()
        apps.get_model("extras", "Status").objects.all().delete()

        apps.get_model("ipam", "Namespace").objects.get_or_create(
            pk="26756c2d-fddd-4128-9f88-dbcbddcbef45",
            name="Global",
            description="Default Global namespace. Created by Nautobot.",
        )

        mute_diffsync_logging()
        # pylint: disable=invalid-name
        self.maxDiff = None

    def _import(self, fixtures_name: str, fixtures_path: Path):
        """Test import.

        This method is called by each dynamically-created test case.

        Runs import twice, first time to import data and the second time to verify that nothing has changed.
        """
        input_ref = _INPUTS.get(fixtures_name, fixtures_path / "input.json")

        expected_summary = ImportSummary()
        try:
            expected_summary.load(fixtures_path / "summary.json")
        # pylint: disable=broad-exception-caught
        except Exception:
            if not _BUILD_FIXTURES:
                raise
            # Allow to generate summary
            expected_summary = None

        # Import the file to fresh Nautobot instance
        source = self._import_file(input_ref)

        # Build summary in text format in all cases to allow comparison
        source.summary.dump(fixtures_path / "summary.txt", output_format="text")
        if _BUILD_FIXTURES:
            source.summary.dump(fixtures_path / "summary.json")
        if not expected_summary:
            expected_summary = source.summary

        expected_diffsync_summary = {
            "create": 0,
            "skip": 0,
            "no-change": 0,
            "delete": 0,
            "update": 0,
        }

        self.assertEqual(len(expected_summary.source), len(source.summary.source), "Source model counts mismatch")

        for expected_item in expected_summary.source:
            source_item = next(
                item for item in source.summary.source if item.content_type == expected_item.content_type
            )
            self.assertEqual(
                expected_item.stats.__dict__,
                source_item.stats.__dict__,
                f"Source model stats mismatch for {expected_item.content_type}",
            )

        self.assertEqual(len(expected_summary.nautobot), len(source.summary.nautobot), "Nautobot model counts mismatch")

        save_failed_sum = 0
        for expected_item in expected_summary.nautobot:
            nautobot_item = next(
                item for item in source.summary.nautobot if item.content_type == expected_item.content_type
            )
            self.assertEqual(
                expected_item.stats.__dict__,
                nautobot_item.stats.__dict__,
                f"Nautobot model stats mismatch for {expected_item.content_type}",
            )
            expected_diffsync_summary["create"] += expected_item.stats.source_created
            expected_diffsync_summary["skip"] += expected_item.stats.source_ignored
            save_failed_sum += nautobot_item.stats.save_failed

        self.assertEqual(expected_summary.diffsync, source.summary.diffsync, "DiffSync summary mismatch")
        self.assertEqual(expected_diffsync_summary, source.summary.diffsync, "Expected DiffSync summary mismatch")

        # Re-import the same file to verify that nothing has changed
        second_source = self._import_file(input_ref)
        expected_diffsync_summary["no-change"] = expected_diffsync_summary["create"] - save_failed_sum
        expected_diffsync_summary["create"] = save_failed_sum
        self.assertEqual(
            expected_diffsync_summary, second_source.summary.diffsync, "Expected DiffSync 2 summary mismatch"
        )

        # Verify data
        samples_path = fixtures_path / "samples"
        if not (samples_path).is_dir():
            samples_path.mkdir(parents=True)

        for wrapper in source.nautobot.wrappers.values():
            if wrapper.stats.source_created > 0:
                with self.subTest(f"Verify data {fixtures_name} {wrapper.content_type}"):
                    self._verify_model(samples_path, wrapper)

        if expected_summary is source.summary:
            self.fail("Expected summary was generated, please re-run the test")

    def _import_file(self, input_ref):
        source = NetBoxAdapter(
            input_ref,
            NetBoxImporterOptions(
                dry_run=False,
                bypass_data_validation=True,
                sitegroup_parent_always_region=True,
            ),
        )
        source.import_to_nautobot()

        return source

    def _verify_model(self, samples_path: Path, wrapper: NautobotModelWrapper):
        """Verify data."""
        self.assertLessEqual(
            wrapper.stats.source_created - wrapper.stats.save_failed,
            wrapper.model.objects.count(),
            f"Nautobot instances count mismatch for {wrapper.content_type}",
        )

        path = samples_path / f"{wrapper.content_type}.json"
        if not path.is_file():
            if _BUILD_FIXTURES:
                _generate_fixtures(wrapper, path)
                self.fail(f"Fixture file was generated, please re-run the test {path}")
            else:
                self.fail(f"Fixture file is missing: {path}")

        samples = json.loads(path.read_text())
        model = wrapper.model
        for sample in samples:
            self.assertEqual(
                sample["model"], wrapper.content_type, f"Content type mismatch for {wrapper.content_type} {sample}"
            )

            uid = sample["pk"]
            instance = model.objects.get(pk=uid)
            formatted = json.loads(serialize("json", [instance], ensure_ascii=False))[0]

            self.assertEqual(uid, formatted["pk"], f"PK mismatch for {wrapper.content_type} {instance}")
            formatted_fields = formatted["fields"]

            for key, value in sample["fields"].items():
                if key.startswith("_") or key in _DONT_COMPARE_FIELDS:
                    continue
                if key == "content_types":
                    self.assertEqual(
                        sorted(value),
                        sorted(formatted_fields[key]),
                        f"Data mismatch for {wrapper.content_type} {uid} {key}",
                    )
                else:
                    self.assertEqual(
                        value, formatted_fields.get(key, ""), f"Data mismatch for {wrapper.content_type} {uid} {key}"
                    )
setUp()

Set up test environment.

Source code in nautobot_netbox_importer/tests/test_import.py
def setUp(self):
    """Set up test environment."""
    super().setUp()

    apps.get_model("extras", "Role").objects.all().delete()
    apps.get_model("extras", "Status").objects.all().delete()

    apps.get_model("ipam", "Namespace").objects.get_or_create(
        pk="26756c2d-fddd-4128-9f88-dbcbddcbef45",
        name="Global",
        description="Default Global namespace. Created by Nautobot.",
    )

    mute_diffsync_logging()
    # pylint: disable=invalid-name
    self.maxDiff = None

utils

Utility functions and classes for nautobot_netbox_importer.

get_field_choices(items)

Yield all choices from a model field, flattening nested iterables.

Source code in nautobot_netbox_importer/utils.py
def get_field_choices(items: Iterable) -> Generator[Tuple[Any, Any], None, None]:
    """Yield all choices from a model field, flattening nested iterables."""
    for key, value in items:
        if isinstance(value, (list, tuple)):
            yield from get_field_choices(value)
        else:
            yield key, value