Inplace Optimize

model_navigator.optimize

optimize(func, dataloader, config=None)

Optimize registered modules executed in scope of callable.

Parameters:

• func (Callable) –

  Callable in scope of which optimize is executed.

• dataloader (Sequence[Tuple[int, Any]]) –

  List of tuples with batch size and input.

• config (Optional[OptimizeConfig], default: None ) –

  Optimize config.

Source code in model_navigator/inplace/__init__.py

def optimize(
    func: Callable,
    dataloader: Sequence[Tuple[int, Any]],
    config: Optional[OptimizeConfig] = None,
) -> InplaceOptimizeStatus:
    """Optimize registered modules executed in scope of callable.

    Args:
        func:  Callable in scope of which optimize is executed.
        dataloader: List of tuples with batch size and input.
        config: Optimize config.
    """
    try:
        with ctx.global_context.temporary() as tmp_ctx:
            tmp_ctx.set(ctx.INPLACE_OPTIMIZE_STRATEGIES_CONTEXT_KEY, inplace_config.strategies)
            tmp_ctx.set(ctx.INPLACE_OPTIMIZE_KEY, True)
            if config is None:
                config = OptimizeConfig()

            for m in module_registry.values():
                # set main config if user did not provide one for a module
                if m.optimize_config is None:
                    m.optimize_config = config
                m.load_recorded()

            for input_ in dataloader:
                batch_size, sample = input_  # unpack batch_size and sample
                tmp_ctx.set(ctx.INPLACE_OPTIMIZE_BATCH_CONTEXT_KEY, batch_size)
                if not isinstance(sample, (list, tuple)):
                    sample = (sample,)
                if not isinstance(sample[-1], dict):
                    sample = (*sample, {})
                *args, kwargs = sample
                func(*args, **kwargs)

            module_registry.optimize()
            return _build_optimize_status()
    except Exception as e:
        raise e

model_navigator.OptimizeConfig dataclass

OptimizeConfig(
    sample_count=DEFAULT_SAMPLE_COUNT,
    batching=True,
    input_names=None,
    output_names=None,
    target_formats=None,
    target_device=CUDA,
    runners=None,
    optimization_profile=None,
    workspace=None,
    verbose=False,
    debug=False,
    verify_func=None,
    custom_configs=None,
    model_precision=None,
)

Configuration for inplace Optimize.

Parameters:

• sample_count (int, default: DEFAULT_SAMPLE_COUNT ) –

  Limits how many samples will be used from dataloader

• batching (Optional[bool], default: True ) –

  Enable or disable batching on first (index 0) dimension of the model

• input_names (Optional[Tuple[str, ...]], default: None ) –

  Model input names

• output_names (Optional[Tuple[str, ...]], default: None ) –

  Model output names

• target_formats (Optional[Tuple[Union[str, Format], ...]], default: None ) –

  Target model formats for optimize process

• target_device (Optional[DeviceKind], default: CUDA ) –

  Target device for optimize process, default is CUDA

• runners (Optional[Tuple[Union[str, Type[NavigatorRunner]], ...]], default: None ) –

  Use only runners provided as parameter

• optimization_profile (Optional[OptimizationProfile], default: None ) –

  Optimization profile for conversion and profiling

• workspace (Optional[Path], default: None ) –

  Workspace where packages will be extracted

• verbose (Optional[bool], default: False ) –

  Enable verbose logging

• debug (Optional[bool], default: False ) –

  Enable debug logging from commands

• verify_func (Optional[VerifyFunction], default: None ) –

  Function for additional model verification

• custom_configs (Optional[Sequence[CustomConfig]], default: None ) –

  Sequence of CustomConfigs used to control produced artifacts

• model_precision (Optional[str], default: None ) –

  Source model precision

clone

clone()

Clone the current OptimizeConfig using deepcopy.

Source code in model_navigator/inplace/config.py

def clone(self) -> "OptimizeConfig":
    """Clone the current OptimizeConfig using deepcopy."""
    return copy.deepcopy(self)

to_dict

to_dict()

Convert OptimizeConfig to dictionary.

Source code in model_navigator/inplace/config.py

def to_dict(self) -> Dict[str, Any]:
    """Convert OptimizeConfig to dictionary."""
    config_dict = {}
    for field in dataclasses.fields(self):
        value = getattr(self, field.name)
        config_dict[field.name] = value
    return config_dict

model_navigator.load_optimized

load_optimized(device='cuda')

Load optimized modules.

Parameters:

• device (Union[str, device], default: 'cuda' ) –

  Device on which optimized models are loaded.

Source code in model_navigator/inplace/__init__.py

def load_optimized(device: Union[str, "torch.device"] = "cuda"):  # noqa: F821
    # pytype: enable=name-error
    """Load optimized modules.

    Args:
        device: Device on which optimized models are loaded.
    """
    for m in module_registry.values():
        m.load_optimized(device=device)