"""EfficientNet Encoder Implementation using timm.
This module provides an implementation of EfficientNet-based encoders for use in
semantic segmentation and other computer vision tasks. It leverages the `timm`
library for model components and adds encoder-specific functionality such as
custom input channels, dilation support, and configurable scaling parameters.
Key Components:
---------------
- EncoderMixin:
Mixin class adding encoder-specific features like output channels and stride.
- EfficientNetBaseEncoder:
Base encoder combining EfficientNet backbone with encoder functionality.
- EfficientNetEncoder:
Configurable EfficientNet encoder supporting depth and channel scaling.
- timm_efficientnet_encoders:
Dictionary of available EfficientNet encoder configurations and pretrained settings.
Features:
---------
- Supports arbitrary input channels (e.g., grayscale or multi-channel images).
- Allows conversion to dilated versions for semantic segmentation.
- Provides pretrained weights from multiple sources (ImageNet, AdvProp, Noisy Student).
- Implements scaling rules for EfficientNet architecture.
Example:
>>> from tiatoolbox.models.architecture.timm_efficientnet import EfficientNetEncoder
>>> encoder = EfficientNetEncoder(
... stage_idxs=[2, 3, 5],
... out_channels=[3, 32, 24, 40, 112, 320],
... channel_multiplier=1.0,
... depth_multiplier=1.0,
... drop_rate=0.2
... )
>>> x = torch.randn(1, 3, 224, 224)
>>> features = encoder(x)
>>> [f.shape for f in features]
[torch.Size([1, 3, 224, 224]), torch.Size([1, 32, 112, 112]), ...]
References:
- Tan, Mingxing, and Quoc V. Le. "EfficientNet: Rethinking Model Scaling for
Convolutional Neural Networks." arXiv preprint arXiv:1905.11946 (2019).
URL: https://arxiv.org/abs/1905.11946
"""
from __future__ import annotations
from functools import partial
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING: # pragma: no cover
from collections.abc import Mapping, Sequence
import torch
from timm.layers.activations import Swish
from timm.models._efficientnet_builder import decode_arch_def, round_channels
from timm.models.efficientnet import EfficientNet
from tiatoolbox.models.architecture.utils import (
EncoderMixin,
)
MAX_DEPTH = 5
MIN_DEPTH = 1
DEFAULT_IN_CHANNELS = 3
[docs]
def get_efficientnet_kwargs(
channel_multiplier: float = 1.0,
depth_multiplier: float = 1.0,
drop_rate: float = 0.2,
) -> dict[str, Any]:
"""Generate configuration parameters for EfficientNet.
Args:
channel_multiplier (float):
Multiplier for number of channels per layer. Defaults to 1.0.
depth_multiplier (float):
Multiplier for number of repeats per stage. Defaults to 1.0.
drop_rate (float):
Dropout rate. Defaults to 0.2.
Reference implementation:
https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py
Paper:
https://arxiv.org/abs/1905.11946
EfficientNet parameters:
- 'efficientnet-b0': (1.0, 1.0, 224, 0.2)
- 'efficientnet-b1': (1.0, 1.1, 240, 0.2)
- 'efficientnet-b2': (1.1, 1.2, 260, 0.3)
- 'efficientnet-b3': (1.2, 1.4, 300, 0.3)
- 'efficientnet-b4': (1.4, 1.8, 380, 0.4)
- 'efficientnet-b5': (1.6, 2.2, 456, 0.4)
- 'efficientnet-b6': (1.8, 2.6, 528, 0.5)
- 'efficientnet-b7': (2.0, 3.1, 600, 0.5)
- 'efficientnet-b8': (2.2, 3.6, 672, 0.5)
- 'efficientnet-l2': (4.3, 5.3, 800, 0.5)
Args:
channel_multiplier: Multiplier to number of channels per layer. Defaults to 1.0.
depth_multiplier: Multiplier to number of repeats per stage. Defaults to 1.0.
drop_rate: Dropout rate. Defaults to 0.2.
Returns:
dict[str, Any]:
Dictionary containing EfficientNet configuration parameters
Example:
>>> kwargs = get_efficientnet_kwargs(
... channel_multiplier=1.2,
... depth_multiplier=1.4,
... )
"""
arch_def = [
["ds_r1_k3_s1_e1_c16_se0.25"],
["ir_r2_k3_s2_e6_c24_se0.25"],
["ir_r2_k5_s2_e6_c40_se0.25"],
["ir_r3_k3_s2_e6_c80_se0.25"],
["ir_r3_k5_s1_e6_c112_se0.25"],
["ir_r4_k5_s2_e6_c192_se0.25"],
["ir_r1_k3_s1_e6_c320_se0.25"],
]
return {
"block_args": decode_arch_def(arch_def, depth_multiplier),
"num_features": round_channels(1280, channel_multiplier, 8, None),
"stem_size": 32,
"round_chs_fn": partial(round_channels, multiplier=channel_multiplier),
"act_layer": Swish,
"drop_rate": drop_rate,
"drop_path_rate": 0.2,
}
[docs]
class EfficientNetBaseEncoder(EfficientNet, EncoderMixin):
"""Base class for EfficientNet encoder.
Combines EfficientNet backbone from `timm` with encoder-specific functionality
for feature extraction in segmentation and classification tasks.
Features:
- Supports configurable depth and output stride.
- Provides intermediate feature maps for multi-scale processing.
- Removes classifier for encoder-only usage.
Raises:
ValueError:
If `depth` is not in range [1, 5].
Example:
>>> encoder = EfficientNetBaseEncoder(
... stage_idxs=[2, 3, 5],
... out_channels=[3, 32, 24, 40, 112, 320],
... depth=5,
... output_stride=32
... )
>>> x = torch.randn(1, 3, 224, 224)
>>> features = encoder(x)
>>> [f.shape for f in features]
... [torch.Size([1, 3, 224, 224]), torch.Size([1, 32, 112, 112]), ...]
"""
def __init__(
self,
stage_idxs: list[int],
out_channels: list[int],
depth: int = 5,
output_stride: int = 32,
**kwargs: dict[str, Any],
) -> None:
"""Initialize EfficientNetBaseEncoder.
Args:
stage_idxs (list[int]):
Indices of stages for feature extraction.
out_channels (list[int]):
Output channels for each depth level.
depth (int):
Encoder depth (1-5). Defaults to 5.
output_stride (int):
Output stride of encoder. Defaults to 32.
**kwargs (dict[str, Any]):
Additional keyword arguments for EfficientNet initialization.
Raises:
ValueError:
If `depth` is not in range [1, 5].
"""
if depth > MAX_DEPTH or depth < MIN_DEPTH:
msg = f"{self.__class__.__name__} depth should be in range \
[1, 5], got {depth}"
raise ValueError(msg)
super().__init__(**kwargs)
self._stage_idxs = stage_idxs
self._depth = depth
self._in_channels = 3
self._out_channels = out_channels
self._output_stride = output_stride
del self.classifier
[docs]
def get_stages(self) -> dict[int, Sequence[torch.nn.Module]]:
"""Return encoder stages for dilation modification.
Provides mapping of output strides to corresponding module sequences,
enabling conversion to dilated versions for segmentation tasks.
Returns:
dict[int, Sequence[torch.nn.Module]]:
Dictionary mapping output stride to module sequences.
Example:
>>> stages = encoder.get_stages()
>>> print(stages.keys())
... dict_keys([16, 32])
"""
return {
16: [self.blocks[self._stage_idxs[1] : self._stage_idxs[2]]], # type: ignore[attr-defined]
32: [self.blocks[self._stage_idxs[2] :]], # type: ignore[attr-defined]
}
[docs]
def forward(self, x: torch.Tensor) -> list[torch.Tensor]:
"""Forward pass through EfficientNet encoder.
Extracts feature maps from multiple stages of the encoder for use in
decoder networks or multi-scale processing.
Args:
x (torch.Tensor):
Input tensor of shape (N, C, H, W).
Returns:
list[torch.Tensor]:
List of feature maps from different encoder depths.
Example:
>>> x = torch.randn(1, 3, 224, 224)
>>> features = encoder(x)
>>> len(features)
... 6
"""
features = [x]
if self._depth >= 1:
x = self.conv_stem(x) # type: ignore[attr-defined]
x = self.bn1(x) # type: ignore[attr-defined]
features.append(x)
if self._depth >= 2: # noqa: PLR2004
x = self.blocks[0](x) # type: ignore[attr-defined]
x = self.blocks[1](x) # type: ignore[attr-defined]
features.append(x)
if self._depth >= 3: # noqa: PLR2004
x = self.blocks[2](x) # type: ignore[attr-defined]
features.append(x)
if self._depth >= 4: # noqa: PLR2004
x = self.blocks[3](x) # type: ignore[attr-defined]
x = self.blocks[4](x) # type: ignore[attr-defined]
features.append(x)
if self._depth >= 5: # noqa: PLR2004
x = self.blocks[5](x) # type: ignore[attr-defined]
x = self.blocks[6](x) # type: ignore[attr-defined]
features.append(x)
return features
[docs]
def load_state_dict(
self,
state_dict: Mapping[str, Any],
**kwargs: bool,
) -> torch.nn.modules.module._IncompatibleKeys:
"""Load state dictionary, excluding classifier weights.
Removes classifier weights from the state dictionary before loading,
as the encoder does not include a classification head.
Args:
state_dict (Mapping[str, Any]):
State dictionary to load.
**kwargs (bool):
Additional keyword arguments for `load_state_dict`.
Returns:
torch.nn.modules.module._IncompatibleKeys:
Result of parent class `load_state_dict` method.
Example:
>>> encoder.load_state_dict(torch.load("efficientnet_weights.pth"))
"""
# Create a mutable copy of the state dict to modify
state_dict_copy = dict(state_dict)
state_dict_copy.pop("classifier.bias", None)
state_dict_copy.pop("classifier.weight", None)
return super().load_state_dict(state_dict_copy, **kwargs)
[docs]
class EfficientNetEncoder(EfficientNetBaseEncoder):
"""EfficientNet encoder with configurable scaling parameters.
This class extends `EfficientNetBaseEncoder` to provide scaling options
for depth and channel multipliers, enabling flexible encoder configurations
for segmentation and classification tasks.
Features:
- Supports depth and channel scaling.
- Provides pretrained weights for multiple variants.
- Outputs multi-scale feature maps for downstream tasks.
Example:
>>> encoder = EfficientNetEncoder(
... stage_idxs=[2, 3, 5],
... out_channels=[3, 32, 24, 40, 112, 320],
... channel_multiplier=1.0,
... depth_multiplier=1.0,
... drop_rate=0.2
... )
>>> x = torch.randn(1, 3, 224, 224)
>>> features = encoder(x)
>>> [f.shape for f in features]
... [torch.Size([1, 3, 224, 224]), torch.Size([1, 32, 112, 112]), ...]
"""
def __init__(
self,
stage_idxs: list[int],
out_channels: list[int],
depth: int = 5,
channel_multiplier: float = 1.0,
depth_multiplier: float = 1.0,
drop_rate: float = 0.2,
output_stride: int = 32,
) -> None:
"""Initialize EfficientNetEncoder.
Creates an EfficientNet encoder with configurable scaling parameters
for depth and channel multipliers.
Args:
stage_idxs (list[int]):
Indices of stages for feature extraction.
out_channels (list[int]):
Output channels for each depth level.
depth (int):
Encoder depth (1-5). Defaults to 5.
channel_multiplier (float):
Channel scaling factor. Defaults to 1.0.
depth_multiplier (float):
Depth scaling factor. Defaults to 1.0.
drop_rate (float):
Dropout rate. Defaults to 0.2.
output_stride (int):
Output stride of encoder. Defaults to 32.
"""
kwargs = get_efficientnet_kwargs(
channel_multiplier, depth_multiplier, drop_rate
)
super().__init__(
stage_idxs=stage_idxs,
depth=depth,
out_channels=out_channels,
output_stride=output_stride,
**kwargs,
)
timm_efficientnet_encoders = {
"timm-efficientnet-b0": {
"encoder": EfficientNetEncoder,
"pretrained_settings": {
"imagenet": {
"repo_id": "smp-hub/timm-efficientnet-b0.imagenet",
"revision": "8419e9cc19da0b68dcd7bb12f19b7c92407ad7c4",
},
"advprop": {
"repo_id": "smp-hub/timm-efficientnet-b0.advprop",
"revision": "a5870af2d24ce79e0cc7fae2bbd8e0a21fcfa6d8",
},
"noisy-student": {
"repo_id": "smp-hub/timm-efficientnet-b0.noisy-student",
"revision": "bea8b0ff726a50e48774d2d360c5fb1ac4815836",
},
},
"params": {
"out_channels": [3, 32, 24, 40, 112, 320],
"stage_idxs": [2, 3, 5],
"channel_multiplier": 1.0,
"depth_multiplier": 1.0,
"drop_rate": 0.2,
},
},
}