Skip to content

Config

Classes, enums and types used to configure the Triton Model Navigator.

model_navigator.configuration

Definition of enums and classes representing configuration for Model Navigator.

CustomConfig

Bases: ABC

Base class used for custom configs. Input for Model Navigator optimize method.

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
534
535
536
def defaults(self) -> None:
    """Update parameters to defaults."""
    return None

from_dict classmethod 

from_dict(config_dict)

Instantiate CustomConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
538
539
540
541
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "CustomConfig":
    """Instantiate CustomConfig from a dictionary."""
    return cls(**config_dict)

name abstractmethod classmethod 

name()

Name of the CustomConfig.

Source code in model_navigator/configuration/__init__.py 
528
529
530
531
532
@classmethod
@abc.abstractmethod
def name(cls) -> str:
    """Name of the CustomConfig."""
    raise NotImplementedError()

CustomConfigForFormat dataclass 

CustomConfigForFormat(custom_args=dict(), device=None)

Bases: DataObject, CustomConfig

Abstract base class used for custom configs representing particular format.

Parameters:

  • custom_args (Dict[str, Any], default: dict() ) –

    Custom arguments passed to conversion function.

  • device (Optional[str], default: None ) –

    torch-like string used for selecting device e.q. "cuda:0".

format abstractmethod property 

format

Format represented by CustomConfig.

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
534
535
536
def defaults(self) -> None:
    """Update parameters to defaults."""
    return None

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate CustomConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
538
539
540
541
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "CustomConfig":
    """Instantiate CustomConfig from a dictionary."""
    return cls(**config_dict)

name abstractmethod classmethod 

name()

Name of the CustomConfig.

Source code in model_navigator/configuration/__init__.py 
528
529
530
531
532
@classmethod
@abc.abstractmethod
def name(cls) -> str:
    """Name of the CustomConfig."""
    raise NotImplementedError()

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

CustomConfigForTensorRT dataclass 

CustomConfigForTensorRT(custom_args=dict(), device=None, trt_profiles=None, trt_profile=None, precision=DEFAULT_TENSORRT_PRECISION, precision_mode=DEFAULT_TENSORRT_PRECISION_MODE, max_workspace_size=DEFAULT_MAX_WORKSPACE_SIZE, run_max_batch_size_search=None)

Bases: CustomConfigForFormat

Abstract base class used for custom configs representing particular TensorRT format.

format abstractmethod property 

format

Format represented by CustomConfig.

__post_init__ 

__post_init__()

Initialize common TensorRT parameters and validate configuration.

Source code in model_navigator/configuration/__init__.py 
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
def __post_init__(self):
    """Initialize common TensorRT parameters and validate configuration."""
    precision = (self.precision,) if not isinstance(self.precision, (list, tuple)) else self.precision
    self.precision = tuple(TensorRTPrecision(p) for p in precision)
    self.precision_mode = TensorRTPrecisionMode(self.precision_mode)

    # TODO: Remove before 1.0.0 release
    if self.trt_profile is not None and self.trt_profiles is not None:
        raise ModelNavigatorConfigurationError("Only one of trt_profile and trt_profiles can be set.")
    elif self.trt_profile:
        warnings.warn(
            "trt_profile will be deprecated in future releases. Use trt_profiles instead.",
            category=DeprecationWarning,
            stacklevel=2,
        )
        self.trt_profiles = [self.trt_profile]

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
591
592
593
594
595
596
597
598
def defaults(self) -> None:
    """Update parameters to defaults."""
    self.precision = tuple(TensorRTPrecision(p) for p in DEFAULT_TENSORRT_PRECISION)
    self.precision_mode = DEFAULT_TENSORRT_PRECISION_MODE
    self.max_workspace_size = DEFAULT_MAX_WORKSPACE_SIZE
    self.trt_profiles = None
    self.trt_profile = None
    self.run_max_batch_size_search = None

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate CustomConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
538
539
540
541
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "CustomConfig":
    """Instantiate CustomConfig from a dictionary."""
    return cls(**config_dict)

name abstractmethod classmethod 

name()

Name of the CustomConfig.

Source code in model_navigator/configuration/__init__.py 
528
529
530
531
532
@classmethod
@abc.abstractmethod
def name(cls) -> str:
    """Name of the CustomConfig."""
    raise NotImplementedError()

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

DeviceKind

Bases: Enum

Supported types of devices.

Parameters:

  • CPU (str) –

    Select CPU device.

  • GPU (str) –

    Select GPU with CUDA support.

Format

Bases: Enum

All model formats supported by Model Navigator 'optimize' function.

Parameters:

  • PYTHON (str) –

    Format indicating any model defined in Python.

  • TORCH (str) –

    Format indicating PyTorch model.

  • TENSORFLOW (str) –

    Format indicating TensorFlow model.

  • JAX (str) –

    Format indicating JAX model.

  • TORCHSCRIPT (str) –

    Format indicating TorchScript model.

  • TF_SAVEDMODEL (str) –

    Format indicating TensorFlow SavedModel.

  • TF_TRT (str) –

    Format indicating TensorFlow TensorRT model.

  • TORCH_TRT (str) –

    Format indicating PyTorch TensorRT model.

  • ONNX (str) –

    Format indicating ONNX model.

  • TENSORRT (str) –

    Format indicating TensorRT model.

JitType

Bases: Enum

TorchScript export parameter.

Used for selecting the type of TorchScript export.

Parameters:

  • TRACE (str) –

    Use tracing during export.

  • SCRIPT (str) –

    Use scripting during export.

MaxThroughputAndMinLatencyStrategy

Bases: RuntimeSearchStrategy

Get runtime with the highest throughput and the lowest latency.

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
968
969
970
def __str__(self):
    """Return name of strategy."""
    return self.__class__.__name__

MaxThroughputStrategy

Bases: RuntimeSearchStrategy

Get runtime with the highest throughput.

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
968
969
970
def __str__(self):
    """Return name of strategy."""
    return self.__class__.__name__

MaxThroughputWithLatencyBudgetStrategy 

MaxThroughputWithLatencyBudgetStrategy(latency_budget)

Bases: RuntimeSearchStrategy

Get runtime with the hightest throughput within the latency budget.

Initialize the class.

Parameters:

  • latency_budget (float) –

    Latency budget in milliseconds.

Source code in model_navigator/configuration/__init__.py 
 994
 995
 996
 997
 998
 999
1000
1001
def __init__(self, latency_budget: float) -> None:
    """Initialize the class.

    Args:
        latency_budget: Latency budget in milliseconds.
    """
    super().__init__()
    self.latency_budget = latency_budget

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
1003
1004
1005
def __str__(self):
    """Return name of strategy."""
    return f"{self.__class__.__name__}({self.latency_budget}[ms])"

MinLatencyStrategy

Bases: RuntimeSearchStrategy

Get runtime with the lowest latency.

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
968
969
970
def __str__(self):
    """Return name of strategy."""
    return self.__class__.__name__

OnnxConfig dataclass 

OnnxConfig(custom_args=dict(), device=None, opset=DEFAULT_ONNX_OPSET, dynamo_export=False, dynamic_axes=None, dynamo_dynamic_shapes=None, onnx_extended_conversion=False, graph_surgeon_optimization=True, export_device=None, model_path=None)

Bases: CustomConfigForFormat

ONNX custom config used for ONNX export and conversion.

Parameters:

  • opset (Optional[int], default: DEFAULT_ONNX_OPSET ) –

    ONNX opset used for conversion.

  • dynamo_export (Optional[bool], default: False ) –

    Enable additional dynamo export.

  • dynamic_axes (Optional[Dict[str, Union[Dict[int, str], List[int]]]], default: None ) –

    Dynamic axes for ONNX conversion.

  • dynamo_dynamic_shapes (Optional[bool], default: None ) –

    Enable dynamic shapes for dynamo export. By default dynamic shapes are enabled if dynamic_axes are set or batching is enabled.

  • onnx_extended_conversion (bool, default: False ) –

    Enables additional conversions from TorchScript to ONNX.

  • graph_surgeon_optimization (bool, default: True ) –

    Enables polygraphy graph surgeon optimization: fold_constants, infer_shapes, toposort, cleanup.

  • export_device (Optional[str], default: None ) –

    Device used for ONNX export.

  • model_path (Optional[Union[str, Path]], default: None ) –

    optional path to onnx model file, if provided the model will be loaded from the file instead of exporting to onnx

format property 

format

Returns Format.ONNX.

Returns:

defaults 

defaults()

Update parameters to defaults.

Only configuration related to ONNX export and conversion parameters are updated. We leave the dynamo and extended conversion flags as are set during config initialization.

Source code in model_navigator/configuration/__init__.py 
855
856
857
858
859
860
861
862
863
864
865
def defaults(self) -> None:
    """Update parameters to defaults.

    Only configuration related to ONNX export and conversion parameters are updated. We leave the dynamo and
    extended conversion flags as are set during config initialization.
    """
    super().defaults()
    self.opset = DEFAULT_ONNX_OPSET
    self.graph_surgeon_optimization = True
    self.export_device = None
    self.model_path = None

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate CustomConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
538
539
540
541
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "CustomConfig":
    """Instantiate CustomConfig from a dictionary."""
    return cls(**config_dict)

name classmethod 

name()

Name of the config.

Source code in model_navigator/configuration/__init__.py 
850
851
852
853
@classmethod
def name(cls) -> str:
    """Name of the config."""
    return "Onnx"

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

OptimizationProfile dataclass 

OptimizationProfile(max_batch_size=None, batch_sizes=None, window_size=DEFAULT_WINDOW_SIZE, stability_percentage=DEFAULT_STABILITY_PERCENTAGE, stabilization_windows=DEFAULT_STABILIZATION_WINDOWS, min_trials=DEFAULT_MIN_TRIALS, max_trials=DEFAULT_MAX_TRIALS, throughput_cutoff_threshold=DEFAULT_THROUGHPUT_CUTOFF_THRESHOLD, throughput_backoff_limit=DEFAULT_THROUGHPUT_BACKOFF_LIMIT, dataloader=None)

Bases: DataObject

Optimization profile configuration.

For each batch size profiler will run measurements in windows of fixed number of queries. Batch sizes are profiled in the ascending order.

Profiler will run multiple trials and will stop when the measurements are stable (within stability_percentage from the mean) within three consecutive windows. If the measurements are not stable after max_trials trials, the profiler will stop with an error. Profiler will also stop profiling when the throughput does not increase at least by throughput_cutoff_threshold.

Parameters:

  • max_batch_size (Optional[int], default: None ) –

    Maximal batch size used during conversion and profiling. None mean automatic search is enabled.

  • batch_sizes –

    List of batch sizes to profile. None mean automatic search is enabled.

  • window_size (int, default: DEFAULT_WINDOW_SIZE ) –

    Number of requests to measure in each window.

  • stability_percentage (float, default: DEFAULT_STABILITY_PERCENTAGE ) –

    Allowed percentage of variation from the mean in consecutive windows.

  • stabilization_windows (int, default: DEFAULT_STABILIZATION_WINDOWS ) –

    Number consecutive windows selected for stabilization.

  • min_trials (int, default: DEFAULT_MIN_TRIALS ) –

    Minimal number of window trials.

  • max_trials (int, default: DEFAULT_MAX_TRIALS ) –

    Maximum number of window trials.

  • throughput_cutoff_threshold (Optional[float], default: DEFAULT_THROUGHPUT_CUTOFF_THRESHOLD ) –

    Minimum throughput increase to continue profiling.

  • throughput_backoff_limit (int, default: DEFAULT_THROUGHPUT_BACKOFF_LIMIT ) –

    Back-off limit to run multiple more profiling steps to avoid stop at local minimum when throughput saturate based on throughput_cutoff_threshold.

  • dataloader (Optional[SizedDataLoader], default: None ) –

    Optional dataloader for profiling. Use only 1 sample.

__post_init__ 

__post_init__()

Validate OptimizationProfile definition to avoid unsupported configurations.

Source code in model_navigator/configuration/__init__.py 
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
def __post_init__(self):
    """Validate OptimizationProfile definition to avoid unsupported configurations."""
    if self.stability_percentage <= 0:
        raise ModelNavigatorConfigurationError("`stability_percentage` must be greater than 0.0.")

    if self.throughput_backoff_limit < 0:
        raise ModelNavigatorConfigurationError("`throughput_backoff_limit` must be greater then or equal to 0.")

    greater_or_equal_1 = [
        "window_size",
        "stability_percentage",
        "stabilization_windows",
        "min_trials",
        "max_trials",
    ]
    for member in greater_or_equal_1:
        value = getattr(self, member)
        if value < 1:
            raise ModelNavigatorConfigurationError(f"`{member}` must be greater or equal 1.")

    if self.min_trials < self.stabilization_windows:
        raise ModelNavigatorConfigurationError(
            "`min_trials` must be greater or equal than `stabilization_windows`."
        )

    if self.min_trials > self.max_trials:
        raise ModelNavigatorConfigurationError("`max_trials` must be greater or equal `min_trials`.")

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(optimization_profile_dict)

Instantiate OptimizationProfile class from a dictionary.

Parameters:

  • optimization_profile_dict (Mapping) –

    Data dictionary.

Returns:

Source code in model_navigator/configuration/__init__.py 
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
@classmethod
def from_dict(cls, optimization_profile_dict: Mapping) -> "OptimizationProfile":
    """Instantiate OptimizationProfile class from a dictionary.

    Args:
        optimization_profile_dict (Mapping): Data dictionary.

    Returns:
        OptimizationProfile
    """
    return cls(
        max_batch_size=optimization_profile_dict.get("max_batch_size"),
        batch_sizes=optimization_profile_dict.get("batch_sizes"),
        window_size=optimization_profile_dict.get("window_size", DEFAULT_WINDOW_SIZE),
        stability_percentage=optimization_profile_dict.get("stability_percentage", DEFAULT_STABILITY_PERCENTAGE),
        stabilization_windows=optimization_profile_dict.get("stabilization_windows", DEFAULT_STABILIZATION_WINDOWS),
        min_trials=optimization_profile_dict.get("min_trials", DEFAULT_MIN_TRIALS),
        max_trials=optimization_profile_dict.get("max_trials", DEFAULT_MAX_TRIALS),
        throughput_cutoff_threshold=optimization_profile_dict.get(
            "throughput_cutoff_threshold", DEFAULT_THROUGHPUT_CUTOFF_THRESHOLD
        ),
        throughput_backoff_limit=optimization_profile_dict.get(
            "throughput_backoff_limit", DEFAULT_THROUGHPUT_BACKOFF_LIMIT
        ),
    )

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Append dataloader field to filtered fields during dump.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/configuration/__init__.py 
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Append `dataloader` field to filtered fields during dump.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if not filter_fields:
        filter_fields = []

    filter_fields += ["dataloader"]
    return super().to_dict(filter_fields=filter_fields, parse=parse)

RuntimeSearchStrategy

Base class for runtime search strategies.

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
968
969
970
def __str__(self):
    """Return name of strategy."""
    return self.__class__.__name__

SelectedRuntimeStrategy 

SelectedRuntimeStrategy(model_key, runner_name)

Bases: RuntimeSearchStrategy

Get a selected runtime.

Initialize the class.

Parameters:

  • model_key (str) –

    Unique key of the model.

  • runner_name (str) –

    Name of the runner.

Source code in model_navigator/configuration/__init__.py 
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
def __init__(self, model_key: str, runner_name: str) -> None:
    """Initialize the class.

    Args:
        model_key (str): Unique key of the model.
        runner_name (str): Name of the runner.
    """
    super().__init__()
    self.model_key = model_key
    self.runner_name = runner_name

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
1022
1023
1024
def __str__(self):
    """Return name of strategy."""
    return f"{self.__class__.__name__}({self.model_key}:{self.runner_name})"

ShapeTuple dataclass 

ShapeTuple(min, opt, max)

Bases: DataObject

Represents a set of shapes for a single binding in a profile.

Each element of the tuple represents a shape for a single dimension of the binding.

Parameters:

  • min (Tuple[int]) –

    The minimum shape that the profile will support.

  • opt (Tuple[int]) –

    The shape for which TensorRT will optimize the engine.

  • max (Tuple[int]) –

    The maximum shape that the profile will support.

__iter__ 

__iter__()

Iterate over shapes.

Source code in model_navigator/configuration/__init__.py 
218
219
220
def __iter__(self):
    """Iterate over shapes."""
    yield from [self.min, self.opt, self.max]

__repr__ 

__repr__()

Representation.

Source code in model_navigator/configuration/__init__.py 
214
215
216
def __repr__(self):
    """Representation."""
    return type(self).__name__ + self.__str__()

__str__ 

__str__()

String representation.

Source code in model_navigator/configuration/__init__.py 
210
211
212
def __str__(self):
    """String representation."""
    return f"(min={self.min}, opt={self.opt}, max={self.max})"

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

SizedIterable

Bases: Protocol

Protocol representing sized iterable. Used by dataloader.

__iter__ 

__iter__()

Magic method iter.

Returns:

Source code in model_navigator/configuration/__init__.py 
80
81
82
83
84
85
86
def __iter__(self) -> Iterator:
    """Magic method __iter__.

    Returns:
        Iterator to next item.
    """
    ...

__len__ 

__len__()

Magic method len.

Returns:

  • int –

    Length of size iterable.

Source code in model_navigator/configuration/__init__.py 
88
89
90
91
92
93
94
def __len__(self) -> int:
    """Magic method __len__.

    Returns:
        Length of size iterable.
    """
    ...

TensorFlowConfig dataclass 

TensorFlowConfig(custom_args=dict(), device=None, jit_compile=(None), enable_xla=(None))

Bases: CustomConfigForFormat

TensorFlow custom config used for SavedModel export.

Parameters:

  • jit_compile (Tuple[Optional[bool], ...], default: (None) ) –

    Enable or Disable jit_compile flag for tf.function wrapper for Jax infer function.

  • enable_xla (Tuple[Optional[bool], ...], default: (None) ) –

    Enable or Disable enable_xla flag for jax2tf converter.

format property 

format

Returns Format.TF_SAVEDMODEL.

Returns:

  • Format –

    Format.TF_SAVEDMODEL

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
628
629
630
631
def defaults(self) -> None:
    """Update parameters to defaults."""
    self.jit_compile = (None,)
    self.enable_xla = (None,)

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate CustomConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
538
539
540
541
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "CustomConfig":
    """Instantiate CustomConfig from a dictionary."""
    return cls(**config_dict)

name classmethod 

name()

Name of the config.

Source code in model_navigator/configuration/__init__.py 
623
624
625
626
@classmethod
def name(cls) -> str:
    """Name of the config."""
    return "TensorFlow"

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

TensorFlowTensorRTConfig dataclass 

TensorFlowTensorRTConfig(custom_args=dict(), device=None, trt_profiles=None, trt_profile=None, precision=DEFAULT_TENSORRT_PRECISION, precision_mode=DEFAULT_TENSORRT_PRECISION_MODE, max_workspace_size=DEFAULT_MAX_WORKSPACE_SIZE, run_max_batch_size_search=None, minimum_segment_size=DEFAULT_MIN_SEGMENT_SIZE)

Bases: CustomConfigForTensorRT

TensorFlow TensorRT custom config used for TensorRT SavedModel export.

Parameters:

  • minimum_segment_size (int, default: DEFAULT_MIN_SEGMENT_SIZE ) –

    Min size of subgraph.

format property 

format

Returns Format.TF_TRT.

Returns:

__post_init__ 

__post_init__()

Initialize common TensorRT parameters and validate configuration.

Source code in model_navigator/configuration/__init__.py 
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
def __post_init__(self):
    """Initialize common TensorRT parameters and validate configuration."""
    precision = (self.precision,) if not isinstance(self.precision, (list, tuple)) else self.precision
    self.precision = tuple(TensorRTPrecision(p) for p in precision)
    self.precision_mode = TensorRTPrecisionMode(self.precision_mode)

    # TODO: Remove before 1.0.0 release
    if self.trt_profile is not None and self.trt_profiles is not None:
        raise ModelNavigatorConfigurationError("Only one of trt_profile and trt_profiles can be set.")
    elif self.trt_profile:
        warnings.warn(
            "trt_profile will be deprecated in future releases. Use trt_profiles instead.",
            category=DeprecationWarning,
            stacklevel=2,
        )
        self.trt_profiles = [self.trt_profile]

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
658
659
660
661
def defaults(self) -> None:
    """Update parameters to defaults."""
    super().defaults()
    self.minimum_segment_size = DEFAULT_MIN_SEGMENT_SIZE

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate TensorFlowTensorRTConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
663
664
665
666
667
668
669
670
671
672
673
674
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "TensorFlowTensorRTConfig":
    """Instantiate TensorFlowTensorRTConfig from a dictionary."""
    if config_dict.get("trt_profiles") is not None:
        # if config_dict.get("trt_profiles") is not None and not isinstance(config_dict["trt_profile"], TensorRTProfile):
        parsed_trt_profiles = []
        for trt_profile in config_dict.get("trt_profiles"):
            if not isinstance(trt_profile, TensorRTProfile):
                trt_profile = TensorRTProfile.from_dict(trt_profile)
            parsed_trt_profiles.append(trt_profile)
        config_dict["trt_profiles"] = parsed_trt_profiles
    return cls(**config_dict)

name classmethod 

name()

Name of the config.

Source code in model_navigator/configuration/__init__.py 
653
654
655
656
@classmethod
def name(cls) -> str:
    """Name of the config."""
    return "TensorFlowTensorRT"

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

TensorRTCompatibilityLevel

Bases: Enum

Compatibility level for TensorRT.

Parameters:

  • AMPERE_PLUS (str) –

    Support AMPERE plus architecture

TensorRTConfig dataclass 

TensorRTConfig(custom_args=dict(), device=None, trt_profiles=None, trt_profile=None, precision=DEFAULT_TENSORRT_PRECISION, precision_mode=DEFAULT_TENSORRT_PRECISION_MODE, max_workspace_size=DEFAULT_MAX_WORKSPACE_SIZE, run_max_batch_size_search=None, optimization_level=None, compatibility_level=None, onnx_parser_flags=None, timing_cache_dir=None, model_path=None)

Bases: CustomConfigForTensorRT

TensorRT custom config used for TensorRT conversion.

Parameters:

  • optimization_level (Optional[int], default: None ) –

    Optimization level for TensorRT conversion. Allowed values are fom 0 to 5. Where default is 3 based on TensorRT API documentation.

  • compatibility_level (Optional[TensorRTCompatibilityLevel], default: None ) –

    Compatibility level for TensorRT conversion.

  • onnx_parser_flags (Optional[List[int]], default: None ) –

    List of TensorRT OnnxParserFlags used for conversion.

  • timing_cache_dir (Optional[Union[str, Path]], default: None ) –

    Storage directory for TRT tactic timing info collected from builder

  • model_path (Optional[Union[str, Path]], default: None ) –

    optional path to trt model file, if provided the model will be loaded from the file instead of converting onnx to trt

format property 

format

Returns Format.TENSORRT.

Returns:

__post_init__ 

__post_init__()

Parse dataclass enums.

Source code in model_navigator/configuration/__init__.py 
887
888
889
890
891
892
893
def __post_init__(self) -> None:
    """Parse dataclass enums."""
    super().__post_init__()
    if self.optimization_level is not None and (self.optimization_level < 0 or self.optimization_level > 5):
        raise ModelNavigatorConfigurationError(
            f"TensorRT `optimization_level` must be between 0 and 5. Provided value: {self.optimization_level}."
        )

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
909
910
911
912
913
914
915
916
def defaults(self) -> None:
    """Update parameters to defaults."""
    super().defaults()
    self.optimization_level = None
    self.compatibility_level = None
    self.onnx_parser_flags = None
    self.timing_cache_dir = None
    self.model_path = None

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate TensorRTConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
918
919
920
921
922
923
924
925
926
927
928
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "TensorRTConfig":
    """Instantiate TensorRTConfig from a dictionary."""
    if config_dict.get("trt_profiles") is not None:
        parsed_trt_profiles = []
        for trt_profile in config_dict.get("trt_profiles"):
            if not isinstance(trt_profile, TensorRTProfile):
                trt_profile = TensorRTProfile.from_dict(trt_profile)
            parsed_trt_profiles.append(trt_profile)
        config_dict["trt_profiles"] = parsed_trt_profiles
    return cls(**config_dict)

name classmethod 

name()

Name of the config.

Source code in model_navigator/configuration/__init__.py 
904
905
906
907
@classmethod
def name(cls) -> str:
    """Name of the config."""
    return "TensorRT"

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

TensorRTPrecision

Bases: Enum

Precisions supported during TensorRT conversions.

Parameters:

  • INT8 (str) –

    8-bit integer precision.

  • FP8 (str) –

    8-bit floating point precision.

  • FP16 (str) –

    16-bit floating point precision.

  • BF16 (str) –

    16-bit brain floating point precision.

  • FP32 (str) –

    32-bit floating point precision.

TensorRTPrecisionMode

Bases: Enum

Precision modes for TensorRT conversions.

Parameters:

  • HIERARCHY (str) –

    Use TensorRT precision hierarchy starting from highest to lowest.

  • SINGLE (str) –

    Use single precision.

  • MIXED (str) –

    Use mixed precision.

TensorRTProfile

Bases: Dict[str, ShapeTuple]

Single optimization profile that can be used to build an engine.

More specifically, it is an Dict[str, ShapeTuple] which maps binding names to a set of min/opt/max shapes.

__getitem__ 

__getitem__(key)

Retrieves the shapes registered for a given input name.

Returns:

  • ShapeTuple – 
    A named tuple including ``min``, ``opt``, and ``max`` members for the shapes
corresponding to the input.
Source code in model_navigator/configuration/__init__.py 
385
386
387
388
389
390
391
392
393
394
395
def __getitem__(self, key):
    """Retrieves the shapes registered for a given input name.

    Returns:
        ShapeTuple:
                A named tuple including ``min``, ``opt``, and ``max`` members for the shapes
                corresponding to the input.
    """
    if key not in self:
        LOGGER.error(f"Binding: {key} does not have shapes set in this profile")
    return super().__getitem__(key)

__repr__ 

__repr__()

Representation.

Source code in model_navigator/configuration/__init__.py 
397
398
399
400
401
402
def __repr__(self):
    """Representation."""
    ret = "TensorRTProfile()"
    for name, (min, opt, max) in self.items():
        ret += f".add('{name}', min={min}, opt={opt}, max={max})"
    return ret

__str__ 

__str__()

String representation.

Source code in model_navigator/configuration/__init__.py 
404
405
406
407
408
409
410
411
def __str__(self):
    """String representation."""
    elems = []
    for name, (min, opt, max) in self.items():
        elems.append(f"{name} [min={min}, opt={opt}, max={max}]")

    sep = ",\n "
    return "{" + sep.join(elems) + "}"

add 

add(name, min, opt, max)

A convenience function to add shapes for a single binding.

Parameters:

  • name (str) –

    The name of the binding.

  • min (Tuple[int]) –

    The minimum shape that the profile will support.

  • opt (Tuple[int]) –

    The shape for which TensorRT will optimize the engine.

  • max (Tuple[int]) –

    The maximum shape that the profile will support.

Returns:

  • Profile –

    self, which allows this function to be easily chained to add multiple bindings, e.g., TensorRTProfile().add(...).add(...)

Source code in model_navigator/configuration/__init__.py 
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
def add(self, name, min, opt, max):
    """A convenience function to add shapes for a single binding.

    Args:
        name (str): The name of the binding.
        min (Tuple[int]): The minimum shape that the profile will support.
        opt (Tuple[int]): The shape for which TensorRT will optimize the engine.
        max (Tuple[int]): The maximum shape that the profile will support.

    Returns:
        Profile:
            self, which allows this function to be easily chained to add multiple bindings,
            e.g., TensorRTProfile().add(...).add(...)
    """
    self[name] = ShapeTuple(min, opt, max)
    return self

from_dict classmethod 

from_dict(profile_dict)

Create a TensorRTProfile from a dictionary.

Parameters:

  • profile_dict (Dict[str, Dict[str, Tuple[int, ...]]]) –

    A dictionary mapping binding names to a dictionary containing min, opt, and max keys.

Returns:

  • TensorRTProfile –

    A TensorRTProfile object.

Source code in model_navigator/configuration/__init__.py 
360
361
362
363
364
365
366
367
368
369
370
371
372
373
@classmethod
def from_dict(cls, profile_dict: Dict[str, Dict[str, Tuple[int, ...]]]):
    """Create a TensorRTProfile from a dictionary.

    Args:
        profile_dict (Dict[str, Dict[str, Tuple[int, ...]]]):
            A dictionary mapping binding names to a dictionary containing ``min``, ``opt``, and
            ``max`` keys.

    Returns:
        TensorRTProfile:
            A TensorRTProfile object.
    """
    return cls({name: ShapeTuple(**shapes) for name, shapes in profile_dict.items()})

to_dict 

to_dict()

Serialize to a dictionary.

Returns:

  • Dict[str, Dict[str, Tuple[int, ...]]] –

    Dict[str, Dict[str, Tuple[int, ...]]]: A dictionary mapping binding names to a dictionary containing min, opt, and max keys.

Source code in model_navigator/configuration/__init__.py 
375
376
377
378
379
380
381
382
383
def to_dict(self) -> Dict[str, Dict[str, Tuple[int, ...]]]:
    """Serialize to a dictionary.

    Returns:
        Dict[str, Dict[str, Tuple[int, ...]]]:
            A dictionary mapping binding names to a dictionary containing ``min``, ``opt``, and
            ``max`` keys.
    """
    return {name: vars(shapes) for name, shapes in self.items()}

TensorType

Bases: Enum

All model formats supported by Model Navigator 'optimize' function.

TorchConfig dataclass 

TorchConfig(custom_args=dict(), device=None, autocast=True, inference_mode=True)

Bases: CustomConfigForFormat

Torch custom config used for torch runner.

Parameters:

  • autocast (bool, default: True ) –

    Enable Automatic Mixed Precision in runner (default: False).

  • inference_mode (bool, default: True ) –

    Enable inference mode in runner (default: True).

format property 

format

Returns Format.TORCH.

Returns:

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
703
704
705
706
def defaults(self) -> None:
    """Update parameters to defaults."""
    self.autocast = True
    self.inference_mode = True

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate CustomConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
538
539
540
541
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "CustomConfig":
    """Instantiate CustomConfig from a dictionary."""
    return cls(**config_dict)

name classmethod 

name()

Name of the config.

Source code in model_navigator/configuration/__init__.py 
698
699
700
701
@classmethod
def name(cls) -> str:
    """Name of the config."""
    return "Torch"

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

TorchExportConfig dataclass 

TorchExportConfig(custom_args=dict(), device=None, autocast=True, inference_mode=True)

Bases: CustomConfigForFormat

Torch export custom config used for torch.export.export.

Parameters:

  • autocast (bool, default: True ) –

    Enable Automatic Mixed Precision in runner (default: False).

  • inference_mode (bool, default: True ) –

    Enable inference mode in runner (default: True).

format property 

format

Returns Format.

Returns:

  • Format –

    Format.TORCH_EXPORTEDPROGRAM

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
778
779
780
781
def defaults(self) -> None:
    """Update parameters to defaults."""
    self.autocast = True
    self.inference_mode = True

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate CustomConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
538
539
540
541
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "CustomConfig":
    """Instantiate CustomConfig from a dictionary."""
    return cls(**config_dict)

name classmethod 

name()

Name of the config.

Source code in model_navigator/configuration/__init__.py 
773
774
775
776
@classmethod
def name(cls) -> str:
    """Name of the config."""
    return "TorchExport"

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

TorchScriptConfig dataclass 

TorchScriptConfig(custom_args=dict(), device=None, jit_type=(JitType.SCRIPT, JitType.TRACE), strict=True, autocast=True, inference_mode=True)

Bases: CustomConfigForFormat

Torch custom config used for TorchScript export.

Parameters:

  • jit_type (Union[Union[str, JitType], Tuple[Union[str, JitType], ...]], default: (SCRIPT, TRACE) ) –

    Type of TorchScript export.

  • strict (bool, default: True ) –

    Enable or Disable strict flag for tracer used in TorchScript export (default: True).

  • autocast (bool, default: True ) –

    Enable Automatic Mixed Precision in runner (default: False).

  • inference_mode (bool, default: True ) –

    Enable inference mode in runner (default: True).

format property 

format

Returns Format.TORCHSCRIPT.

Returns:

  • Format –

    Format.TORCHSCRIPT

__post_init__ 

__post_init__()

Parse dataclass enums.

Source code in model_navigator/configuration/__init__.py 
725
726
727
728
def __post_init__(self) -> None:
    """Parse dataclass enums."""
    jit_type = (self.jit_type,) if not isinstance(self.jit_type, (list, tuple)) else self.jit_type
    self.jit_type = tuple(JitType(j) for j in jit_type)

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
744
745
746
747
748
749
def defaults(self) -> None:
    """Update parameters to defaults."""
    self.jit_type = (JitType.SCRIPT, JitType.TRACE)
    self.strict = True
    self.autocast = True
    self.inference_mode = True

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate CustomConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
538
539
540
541
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "CustomConfig":
    """Instantiate CustomConfig from a dictionary."""
    return cls(**config_dict)

name classmethod 

name()

Name of the config.

Source code in model_navigator/configuration/__init__.py 
739
740
741
742
@classmethod
def name(cls) -> str:
    """Name of the config."""
    return "TorchScript"

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

TorchTensorRTConfig dataclass 

TorchTensorRTConfig(custom_args=dict(), device=None, trt_profiles=None, trt_profile=None, precision=DEFAULT_TENSORRT_PRECISION, precision_mode=DEFAULT_TENSORRT_PRECISION_MODE, max_workspace_size=DEFAULT_MAX_WORKSPACE_SIZE, run_max_batch_size_search=None)

Bases: CustomConfigForTensorRT

Torch custom config used for TensorRT TorchScript conversion.

format property 

format

Returns Format.TORCH_TRT.

Returns:

__post_init__ 

__post_init__()

Initialize common TensorRT parameters and validate configuration.

Source code in model_navigator/configuration/__init__.py 
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
def __post_init__(self):
    """Initialize common TensorRT parameters and validate configuration."""
    precision = (self.precision,) if not isinstance(self.precision, (list, tuple)) else self.precision
    self.precision = tuple(TensorRTPrecision(p) for p in precision)
    self.precision_mode = TensorRTPrecisionMode(self.precision_mode)

    # TODO: Remove before 1.0.0 release
    if self.trt_profile is not None and self.trt_profiles is not None:
        raise ModelNavigatorConfigurationError("Only one of trt_profile and trt_profiles can be set.")
    elif self.trt_profile:
        warnings.warn(
            "trt_profile will be deprecated in future releases. Use trt_profiles instead.",
            category=DeprecationWarning,
            stacklevel=2,
        )
        self.trt_profiles = [self.trt_profile]

defaults 

defaults()

Update parameters to defaults.

Source code in model_navigator/configuration/__init__.py 
591
592
593
594
595
596
597
598
def defaults(self) -> None:
    """Update parameters to defaults."""
    self.precision = tuple(TensorRTPrecision(p) for p in DEFAULT_TENSORRT_PRECISION)
    self.precision_mode = DEFAULT_TENSORRT_PRECISION_MODE
    self.max_workspace_size = DEFAULT_MAX_WORKSPACE_SIZE
    self.trt_profiles = None
    self.trt_profile = None
    self.run_max_batch_size_search = None

filter_data staticmethod 

filter_data(data, filter_fields)

Filter fields in dictionary.

Parameters:

  • data (Dict) –

    Dictionary with data to filter

  • filter_fields (List[str]) –

    Fields to filter

Returns:

  • –

    Filtered dictionary

Source code in model_navigator/utils/common.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
@staticmethod
def filter_data(data: Dict, filter_fields: List[str]):
    """Filter fields in dictionary.

    Args:
        data: Dictionary with data to filter
        filter_fields: Fields to filter

    Returns:
        Filtered dictionary
    """
    filtered_data = {key: value for key, value in data.items() if key not in filter_fields}
    return filtered_data

from_dict classmethod 

from_dict(config_dict)

Instantiate TorchTensorRTConfig from a dictionary.

Source code in model_navigator/configuration/__init__.py 
802
803
804
805
806
807
808
809
810
811
812
813
@classmethod
def from_dict(cls, config_dict: Dict[str, Any]) -> "TorchTensorRTConfig":
    """Instantiate TorchTensorRTConfig from a dictionary."""
    if config_dict.get("trt_profiles") is not None:
        # if config_dict.get("trt_profiles") is not None and not isinstance(config_dict["trt_profile"], TensorRTProfile):
        parsed_trt_profiles = []
        for trt_profile in config_dict.get("trt_profiles"):
            if not isinstance(trt_profile, TensorRTProfile):
                trt_profile = TensorRTProfile.from_dict(trt_profile)
            parsed_trt_profiles.append(trt_profile)
        config_dict["trt_profiles"] = parsed_trt_profiles
    return cls(**config_dict)

name classmethod 

name()

Name of the config.

Source code in model_navigator/configuration/__init__.py 
797
798
799
800
@classmethod
def name(cls) -> str:
    """Name of the config."""
    return "TorchTensorRT"

parse_data staticmethod 

parse_data(data)

Parse values in provided data.

Parameters:

  • data (Dict) –

    Dictionary with data to parse

Returns:

  • –

    Parsed dictionary

Source code in model_navigator/utils/common.py 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@staticmethod
def parse_data(data: Dict):
    """Parse values in provided data.

    Args:
        data: Dictionary with data to parse

    Returns:
        Parsed dictionary
    """
    parsed_data = {}
    for key, value in data.items():
        parsed_data[key] = DataObject.parse_value(value)

    return parsed_data

parse_value staticmethod 

parse_value(value)

Parse value to jsonable format.

Parameters:

  • value (Any) –

    Value to be parsed.

Returns:

Source code in model_navigator/utils/common.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@staticmethod
def parse_value(value: Any) -> Union[str, Dict, List]:
    """Parse value to jsonable format.

    Args:
        value (Any): Value to be parsed.

    Returns:
        Union[str, Dict, List]: Jsonable value.
    """
    if isinstance(value, DataObject):
        value = value.to_dict(parse=True)
    elif hasattr(value, "to_json"):
        value = value.to_json()
    elif isinstance(value, (Mapping, Profile)):
        value = DataObject._from_dict(value)
    elif isinstance(value, list) or isinstance(value, tuple):
        value = DataObject._from_list(value)
    elif isinstance(value, Enum):
        value = value.value
    elif isinstance(value, pathlib.Path):
        value = str(value)
    elif isinstance(value, ShapeTuple):
        value = vars(value)

    return value

to_dict 

to_dict(filter_fields=None, parse=False)

Serialize to a dictionary.

Parameters:

  • filter_fields (Optional[List[str]], default: None ) –

    List of fields to filter out. Defaults to None.

  • parse (bool, default: False ) –

    If True recursively parse field values to jsonable representation. Defaults to False.

Returns:

  • Dict ( Dict ) –

    Data serialized to a dictionary.

Source code in model_navigator/utils/common.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def to_dict(self, filter_fields: Optional[List[str]] = None, parse: bool = False) -> Dict:
    """Serialize to a dictionary.

    Args:
        filter_fields (Optional[List[str]], optional): List of fields to filter out.
            Defaults to None.
        parse (bool, optional): If True recursively parse field values to jsonable representation.
            Defaults to False.

    Returns:
        Dict: Data serialized to a dictionary.
    """
    if filter_fields:
        filtered_data = DataObject.filter_data(
            data=self.__dict__,
            filter_fields=filter_fields,
        )
    else:
        filtered_data = self.__dict__

    if parse:
        data = DataObject.parse_data(filtered_data)
    else:
        data = filtered_data

    return data

map_custom_configs 

map_custom_configs(custom_configs)

Map custom configs from list to dictionary.

Parameters:

Returns:

  • Dict –

    Mapped configs to dictionary

Source code in model_navigator/configuration/__init__.py 
931
932
933
934
935
936
937
938
939
940
941
942
943
def map_custom_configs(custom_configs: Optional[Sequence[CustomConfig]]) -> Dict:
    """Map custom configs from list to dictionary.

    Args:
        custom_configs: List of custom configs passed to API method

    Returns:
        Mapped configs to dictionary
    """
    if not custom_configs:
        return {}

    return {config.name(): config for config in custom_configs}

model_navigator.MaxThroughputAndMinLatencyStrategy

Bases: RuntimeSearchStrategy

Get runtime with the highest throughput and the lowest latency.

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
968
969
970
def __str__(self):
    """Return name of strategy."""
    return self.__class__.__name__

model_navigator.MaxThroughputStrategy

Bases: RuntimeSearchStrategy

Get runtime with the highest throughput.

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
968
969
970
def __str__(self):
    """Return name of strategy."""
    return self.__class__.__name__

model_navigator.MinLatencyStrategy

Bases: RuntimeSearchStrategy

Get runtime with the lowest latency.

__str__ 

__str__()

Return name of strategy.

Source code in model_navigator/configuration/__init__.py 
968
969
970
def __str__(self):
    """Return name of strategy."""
    return self.__class__.__name__