Vehicle-cpp/hkpc/vision/references/segmentation/v2_extras.py

"""This file only exists to be lazy-imported and avoid V2-related import warnings when just using V1."""
import torch
from torchvision import tv_tensors
from torchvision.transforms import v2


class PadIfSmaller(v2.Transform):
    def __init__(self, size, fill=0):
        super().__init__()
        self.size = size
        self.fill = v2._utils._setup_fill_arg(fill)

    def _get_params(self, sample):
        _, height, width = v2._utils.query_chw(sample)
        padding = [0, 0, max(self.size - width, 0), max(self.size - height, 0)]
        needs_padding = any(padding)
        return dict(padding=padding, needs_padding=needs_padding)

    def _transform(self, inpt, params):
        if not params["needs_padding"]:
            return inpt

        fill = v2._utils._get_fill(self.fill, type(inpt))
        fill = v2._utils._convert_fill_arg(fill)

        return v2.functional.pad(inpt, padding=params["padding"], fill=fill)


class CocoDetectionToVOCSegmentation(v2.Transform):
    """Turn samples from datasets.CocoDetection into the same format as VOCSegmentation.

    This is achieved in two steps:

    1. COCO differentiates between 91 categories while VOC only supports 21, including background for both. Fortunately,
       the COCO categories are a superset of the VOC ones and thus can be mapped. Instances of the 70 categories not
       present in VOC are dropped and replaced by background.
    2. COCO only offers detection masks, i.e. a (N, H, W) bool-ish tensor, where the truthy values in each individual
       mask denote the instance. However, a segmentation mask is a (H, W) integer tensor (typically torch.uint8), where
       the value of each pixel denotes the category it belongs to. The detection masks are merged into one segmentation
       mask while pixels that belong to multiple detection masks are marked as invalid.
    """

    COCO_TO_VOC_LABEL_MAP = dict(
        zip(
            [0, 5, 2, 16, 9, 44, 6, 3, 17, 62, 21, 67, 18, 19, 4, 1, 64, 20, 63, 7, 72],
            range(21),
        )
    )
    INVALID_VALUE = 255

    def _coco_detection_masks_to_voc_segmentation_mask(self, target):
        if "masks" not in target:
            return None

        instance_masks, instance_labels_coco = target["masks"], target["labels"]

        valid_labels_voc = [
            (idx, label_voc)
            for idx, label_coco in enumerate(instance_labels_coco.tolist())
            if (label_voc := self.COCO_TO_VOC_LABEL_MAP.get(label_coco)) is not None
        ]

        if not valid_labels_voc:
            return None

        valid_voc_category_idcs, instance_labels_voc = zip(*valid_labels_voc)

        instance_masks = instance_masks[list(valid_voc_category_idcs)].to(torch.uint8)
        instance_labels_voc = torch.tensor(instance_labels_voc, dtype=torch.uint8)

        # Calling `.max()` on the stacked detection masks works fine to separate background from foreground as long as
        # there is at most a single instance per pixel. Overlapping instances will be filtered out in the next step.
        segmentation_mask, _ = (instance_masks * instance_labels_voc.reshape(-1, 1, 1)).max(dim=0)
        segmentation_mask[instance_masks.sum(dim=0) > 1] = self.INVALID_VALUE

        return segmentation_mask

    def forward(self, image, target):
        segmentation_mask = self._coco_detection_masks_to_voc_segmentation_mask(target)
        if segmentation_mask is None:
            segmentation_mask = torch.zeros(v2.functional.get_size(image), dtype=torch.uint8)

        return image, tv_tensors.Mask(segmentation_mask)