vkit.engine.char_heatmap.default 

  1# Copyright 2022 vkit-x Administrator. All Rights Reserved.
  2#
  3# This project (vkit-x/vkit) is dual-licensed under commercial and SSPL licenses.
  4#
  5# The commercial license gives you the full rights to create and distribute software
  6# on your own terms without any SSPL license obligations. For more information,
  7# please see the "LICENSE_COMMERCIAL.txt" file.
  8#
  9# This project is also available under Server Side Public License (SSPL).
 10# The SSPL licensing is ideal for use cases such as open source projects with
 11# SSPL distribution, student/academic purposes, hobby projects, internal research
 12# projects without external distribution, or other projects where all SSPL
 13# obligations can be met. For more information, please see the "LICENSE_SSPL.txt" file.
 14from typing import Optional
 15
 16import attrs
 17from numpy.random import Generator as RandomGenerator
 18import numpy as np
 19import cv2 as cv
 20
 21from vkit.element import Mask, ScoreMap, ElementSetOperationMode
 22from vkit.engine.interface import (
 23    Engine,
 24    EngineExecutorFactory,
 25    NoneTypeEngineInitResource,
 26)
 27from .type import CharHeatmapEngineRunConfig, CharHeatmap
 28
 29
 30@attrs.define
 31class CharHeatmapDefaultEngineInitConfig:
 32    # Adjust the std. The std gets smaller as the distance factor gets larger.
 33    # The activated area shrinks to the center as the std gets smaller.
 34    # https://colab.research.google.com/drive/1TQ1-BTisMYZHIRVVNpVwDFPviXYMhT7A
 35    gaussian_map_distance_factor: float = 2.25
 36    gaussian_map_char_radius: int = 25
 37    gaussian_map_preserving_score_min: float = 0.8
 38    weight_neutralized_score_map: float = 0.4
 39
 40
 41@attrs.define
 42class CharHeatmapDefaultDebug:
 43    score_map_max: ScoreMap
 44    score_map_min: ScoreMap
 45    char_overlapped_mask: Mask
 46    char_neutralized_score_map: ScoreMap
 47    neutralized_mask: Mask
 48    neutralized_score_map: ScoreMap
 49
 50
 51class CharHeatmapDefaultEngine(
 52    Engine[
 53        CharHeatmapDefaultEngineInitConfig,
 54        NoneTypeEngineInitResource,
 55        CharHeatmapEngineRunConfig,
 56        CharHeatmap,
 57    ]
 58):  # yapf: disable
 59
 60    @classmethod
 61    def get_type_name(cls) -> str:
 62        return 'default'
 63
 64    def generate_np_gaussian_map(self):
 65        char_radius = self.init_config.gaussian_map_char_radius
 66        side_length = char_radius * 2
 67
 68        # Build distances to the center point.
 69        np_offset = np.abs(np.arange(side_length, dtype=np.float32) - char_radius)
 70        np_vert_offset = np.repeat(np_offset[:, None], side_length, axis=1)
 71        np_hori_offset = np.repeat(np_offset[None, :], side_length, axis=0)
 72        np_distance = np.sqrt(np.square(np_vert_offset) + np.square(np_hori_offset))
 73
 74        np_norm_distance = np_distance / char_radius
 75        np_gaussian_map = np.exp(
 76            -0.5 * np.square(self.init_config.gaussian_map_distance_factor * np_norm_distance)
 77        )
 78
 79        # For perspective transformation.
 80        char_begin = 0
 81        char_end = side_length - 1
 82        np_char_src_points = np.asarray(
 83            [
 84                (char_begin, char_begin),
 85                (char_end, char_begin),
 86                (char_end, char_end),
 87                (char_begin, char_end),
 88            ],
 89            dtype=np.float32,
 90        )
 91
 92        return np_gaussian_map, np_char_src_points
 93
 94    def __init__(
 95        self,
 96        init_config: CharHeatmapDefaultEngineInitConfig,
 97        init_resource: Optional[NoneTypeEngineInitResource] = None,
 98    ):
 99        super().__init__(init_config, init_resource)
100
101        self.np_gaussian_map, self.np_char_points = self.generate_np_gaussian_map()
102
103    def run(self, run_config: CharHeatmapEngineRunConfig, rng: RandomGenerator) -> CharHeatmap:
104        height = run_config.height
105        width = run_config.width
106        char_polygons = run_config.char_polygons
107
108        shape = (height, width)
109
110        # Intermediate score maps.
111        score_map_max = ScoreMap.from_shape(shape)
112        score_map_min = ScoreMap.from_shape(shape, value=1.0)
113
114        for char_polygon in char_polygons:
115            # Get transformation matrix.
116            np_trans_mat = cv.getPerspectiveTransform(
117                self.np_char_points,
118                char_polygon.internals.np_self_relative_points,
119                cv.DECOMP_SVD,
120            )
121
122            # Transform gaussian map.
123            char_bounding_box = char_polygon.bounding_box
124
125            np_gaussian_map = cv.warpPerspective(
126                self.np_gaussian_map,
127                np_trans_mat,
128                (char_bounding_box.width, char_bounding_box.height),
129            )
130            score_map = ScoreMap(mat=np_gaussian_map, box=char_bounding_box)
131
132            # Fill score maps.
133            char_polygon.fill_score_map(score_map_max, score_map, keep_max_value=True)
134            char_polygon.fill_score_map(score_map_min, score_map, keep_min_value=True)
135
136        # Set the char overlapped area while preserving score gte threshold.
137        char_overlapped_mask = Mask.from_polygons(
138            shape,
139            char_polygons,
140            ElementSetOperationMode.INTERSECT,
141        )
142
143        preserving_score_min = self.init_config.gaussian_map_preserving_score_min
144        preserving_mask = Mask(mat=(score_map_max.mat >= preserving_score_min).astype(np.uint8))
145
146        neutralized_mask = Mask.from_masks(
147            shape,
148            [
149                char_overlapped_mask,
150                preserving_mask.to_inverted_mask(),
151            ],
152            ElementSetOperationMode.INTERSECT,
153        )
154
155        # Estimate the neutralized score.
156        np_delta: np.ndarray = score_map_max.mat - score_map_min.mat  # type: ignore
157        np_delta = np.clip(np_delta, 0.0, 1.0)
158        char_neutralized_score_map = ScoreMap(mat=np_delta)
159
160        neutralized_score_map = score_map_max.copy()
161        neutralized_mask.fill_score_map(neutralized_score_map, char_neutralized_score_map)
162
163        weight = self.init_config.weight_neutralized_score_map
164        score_map = ScoreMap(
165            mat=((1 - weight) * score_map_max.mat + weight * neutralized_score_map.mat)
166        )
167
168        debug = None
169        if run_config.enable_debug:
170            debug = CharHeatmapDefaultDebug(
171                score_map_max=score_map_max,
172                score_map_min=score_map_min,
173                char_overlapped_mask=char_overlapped_mask,
174                char_neutralized_score_map=char_neutralized_score_map,
175                neutralized_mask=neutralized_mask,
176                neutralized_score_map=neutralized_score_map,
177            )
178
179        return CharHeatmap(score_map=score_map, debug=debug)
180
181
182char_heatmap_default_engine_executor_factory = EngineExecutorFactory(CharHeatmapDefaultEngine)
class CharHeatmapDefaultEngineInitConfig: 
32class CharHeatmapDefaultEngineInitConfig:
33    # Adjust the std. The std gets smaller as the distance factor gets larger.
34    # The activated area shrinks to the center as the std gets smaller.
35    # https://colab.research.google.com/drive/1TQ1-BTisMYZHIRVVNpVwDFPviXYMhT7A
36    gaussian_map_distance_factor: float = 2.25
37    gaussian_map_char_radius: int = 25
38    gaussian_map_preserving_score_min: float = 0.8
39    weight_neutralized_score_map: float = 0.4
CharHeatmapDefaultEngineInitConfig( gaussian_map_distance_factor: float = 2.25, gaussian_map_char_radius: int = 25, gaussian_map_preserving_score_min: float = 0.8, weight_neutralized_score_map: float = 0.4) 
2def __init__(self, gaussian_map_distance_factor=attr_dict['gaussian_map_distance_factor'].default, gaussian_map_char_radius=attr_dict['gaussian_map_char_radius'].default, gaussian_map_preserving_score_min=attr_dict['gaussian_map_preserving_score_min'].default, weight_neutralized_score_map=attr_dict['weight_neutralized_score_map'].default):
3    self.gaussian_map_distance_factor = gaussian_map_distance_factor
4    self.gaussian_map_char_radius = gaussian_map_char_radius
5    self.gaussian_map_preserving_score_min = gaussian_map_preserving_score_min
6    self.weight_neutralized_score_map = weight_neutralized_score_map

Method generated by attrs for class CharHeatmapDefaultEngineInitConfig.

class CharHeatmapDefaultDebug: 
43class CharHeatmapDefaultDebug:
44    score_map_max: ScoreMap
45    score_map_min: ScoreMap
46    char_overlapped_mask: Mask
47    char_neutralized_score_map: ScoreMap
48    neutralized_mask: Mask
49    neutralized_score_map: ScoreMap
CharHeatmapDefaultDebug( score_map_max: vkit.element.score_map.ScoreMap, score_map_min: vkit.element.score_map.ScoreMap, char_overlapped_mask: vkit.element.mask.Mask, char_neutralized_score_map: vkit.element.score_map.ScoreMap, neutralized_mask: vkit.element.mask.Mask, neutralized_score_map: vkit.element.score_map.ScoreMap) 
2def __init__(self, score_map_max, score_map_min, char_overlapped_mask, char_neutralized_score_map, neutralized_mask, neutralized_score_map):
3    self.score_map_max = score_map_max
4    self.score_map_min = score_map_min
5    self.char_overlapped_mask = char_overlapped_mask
6    self.char_neutralized_score_map = char_neutralized_score_map
7    self.neutralized_mask = neutralized_mask
8    self.neutralized_score_map = neutralized_score_map

Method generated by attrs for class CharHeatmapDefaultDebug.

class CharHeatmapDefaultEngine(vkit.engine.interface.Engine[vkit.engine.char_heatmap.default.CharHeatmapDefaultEngineInitConfig, vkit.engine.interface.NoneTypeEngineInitResource, vkit.engine.char_heatmap.type.CharHeatmapEngineRunConfig, vkit.engine.char_heatmap.type.CharHeatmap]): 
 52class CharHeatmapDefaultEngine(
 53    Engine[
 54        CharHeatmapDefaultEngineInitConfig,
 55        NoneTypeEngineInitResource,
 56        CharHeatmapEngineRunConfig,
 57        CharHeatmap,
 58    ]
 59):  # yapf: disable
 60
 61    @classmethod
 62    def get_type_name(cls) -> str:
 63        return 'default'
 64
 65    def generate_np_gaussian_map(self):
 66        char_radius = self.init_config.gaussian_map_char_radius
 67        side_length = char_radius * 2
 68
 69        # Build distances to the center point.
 70        np_offset = np.abs(np.arange(side_length, dtype=np.float32) - char_radius)
 71        np_vert_offset = np.repeat(np_offset[:, None], side_length, axis=1)
 72        np_hori_offset = np.repeat(np_offset[None, :], side_length, axis=0)
 73        np_distance = np.sqrt(np.square(np_vert_offset) + np.square(np_hori_offset))
 74
 75        np_norm_distance = np_distance / char_radius
 76        np_gaussian_map = np.exp(
 77            -0.5 * np.square(self.init_config.gaussian_map_distance_factor * np_norm_distance)
 78        )
 79
 80        # For perspective transformation.
 81        char_begin = 0
 82        char_end = side_length - 1
 83        np_char_src_points = np.asarray(
 84            [
 85                (char_begin, char_begin),
 86                (char_end, char_begin),
 87                (char_end, char_end),
 88                (char_begin, char_end),
 89            ],
 90            dtype=np.float32,
 91        )
 92
 93        return np_gaussian_map, np_char_src_points
 94
 95    def __init__(
 96        self,
 97        init_config: CharHeatmapDefaultEngineInitConfig,
 98        init_resource: Optional[NoneTypeEngineInitResource] = None,
 99    ):
100        super().__init__(init_config, init_resource)
101
102        self.np_gaussian_map, self.np_char_points = self.generate_np_gaussian_map()
103
104    def run(self, run_config: CharHeatmapEngineRunConfig, rng: RandomGenerator) -> CharHeatmap:
105        height = run_config.height
106        width = run_config.width
107        char_polygons = run_config.char_polygons
108
109        shape = (height, width)
110
111        # Intermediate score maps.
112        score_map_max = ScoreMap.from_shape(shape)
113        score_map_min = ScoreMap.from_shape(shape, value=1.0)
114
115        for char_polygon in char_polygons:
116            # Get transformation matrix.
117            np_trans_mat = cv.getPerspectiveTransform(
118                self.np_char_points,
119                char_polygon.internals.np_self_relative_points,
120                cv.DECOMP_SVD,
121            )
122
123            # Transform gaussian map.
124            char_bounding_box = char_polygon.bounding_box
125
126            np_gaussian_map = cv.warpPerspective(
127                self.np_gaussian_map,
128                np_trans_mat,
129                (char_bounding_box.width, char_bounding_box.height),
130            )
131            score_map = ScoreMap(mat=np_gaussian_map, box=char_bounding_box)
132
133            # Fill score maps.
134            char_polygon.fill_score_map(score_map_max, score_map, keep_max_value=True)
135            char_polygon.fill_score_map(score_map_min, score_map, keep_min_value=True)
136
137        # Set the char overlapped area while preserving score gte threshold.
138        char_overlapped_mask = Mask.from_polygons(
139            shape,
140            char_polygons,
141            ElementSetOperationMode.INTERSECT,
142        )
143
144        preserving_score_min = self.init_config.gaussian_map_preserving_score_min
145        preserving_mask = Mask(mat=(score_map_max.mat >= preserving_score_min).astype(np.uint8))
146
147        neutralized_mask = Mask.from_masks(
148            shape,
149            [
150                char_overlapped_mask,
151                preserving_mask.to_inverted_mask(),
152            ],
153            ElementSetOperationMode.INTERSECT,
154        )
155
156        # Estimate the neutralized score.
157        np_delta: np.ndarray = score_map_max.mat - score_map_min.mat  # type: ignore
158        np_delta = np.clip(np_delta, 0.0, 1.0)
159        char_neutralized_score_map = ScoreMap(mat=np_delta)
160
161        neutralized_score_map = score_map_max.copy()
162        neutralized_mask.fill_score_map(neutralized_score_map, char_neutralized_score_map)
163
164        weight = self.init_config.weight_neutralized_score_map
165        score_map = ScoreMap(
166            mat=((1 - weight) * score_map_max.mat + weight * neutralized_score_map.mat)
167        )
168
169        debug = None
170        if run_config.enable_debug:
171            debug = CharHeatmapDefaultDebug(
172                score_map_max=score_map_max,
173                score_map_min=score_map_min,
174                char_overlapped_mask=char_overlapped_mask,
175                char_neutralized_score_map=char_neutralized_score_map,
176                neutralized_mask=neutralized_mask,
177                neutralized_score_map=neutralized_score_map,
178            )
179
180        return CharHeatmap(score_map=score_map, debug=debug)

Abstract base class for generic types.

A generic type is typically declared by inheriting from this class parameterized with one or more type variables. For example, a generic mapping type might be defined as::

class Mapping(Generic[KT, VT]): def __getitem__(self, key: KT) -> VT: ... # Etc.

This class can then be used as follows::

def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: try: return mapping[key] except KeyError: return default

CharHeatmapDefaultEngine( init_config: vkit.engine.char_heatmap.default.CharHeatmapDefaultEngineInitConfig, init_resource: Union[vkit.engine.interface.NoneTypeEngineInitResource, NoneType] = None) 
 95    def __init__(
 96        self,
 97        init_config: CharHeatmapDefaultEngineInitConfig,
 98        init_resource: Optional[NoneTypeEngineInitResource] = None,
 99    ):
100        super().__init__(init_config, init_resource)
101
102        self.np_gaussian_map, self.np_char_points = self.generate_np_gaussian_map()
@classmethod
def get_type_name(cls) -> str: 
61    @classmethod
62    def get_type_name(cls) -> str:
63        return 'default'
def generate_np_gaussian_map(self): 
65    def generate_np_gaussian_map(self):
66        char_radius = self.init_config.gaussian_map_char_radius
67        side_length = char_radius * 2
68
69        # Build distances to the center point.
70        np_offset = np.abs(np.arange(side_length, dtype=np.float32) - char_radius)
71        np_vert_offset = np.repeat(np_offset[:, None], side_length, axis=1)
72        np_hori_offset = np.repeat(np_offset[None, :], side_length, axis=0)
73        np_distance = np.sqrt(np.square(np_vert_offset) + np.square(np_hori_offset))
74
75        np_norm_distance = np_distance / char_radius
76        np_gaussian_map = np.exp(
77            -0.5 * np.square(self.init_config.gaussian_map_distance_factor * np_norm_distance)
78        )
79
80        # For perspective transformation.
81        char_begin = 0
82        char_end = side_length - 1
83        np_char_src_points = np.asarray(
84            [
85                (char_begin, char_begin),
86                (char_end, char_begin),
87                (char_end, char_end),
88                (char_begin, char_end),
89            ],
90            dtype=np.float32,
91        )
92
93        return np_gaussian_map, np_char_src_points
def run( self, run_config: vkit.engine.char_heatmap.type.CharHeatmapEngineRunConfig, rng: numpy.random._generator.Generator) -> vkit.engine.char_heatmap.type.CharHeatmap: 
104    def run(self, run_config: CharHeatmapEngineRunConfig, rng: RandomGenerator) -> CharHeatmap:
105        height = run_config.height
106        width = run_config.width
107        char_polygons = run_config.char_polygons
108
109        shape = (height, width)
110
111        # Intermediate score maps.
112        score_map_max = ScoreMap.from_shape(shape)
113        score_map_min = ScoreMap.from_shape(shape, value=1.0)
114
115        for char_polygon in char_polygons:
116            # Get transformation matrix.
117            np_trans_mat = cv.getPerspectiveTransform(
118                self.np_char_points,
119                char_polygon.internals.np_self_relative_points,
120                cv.DECOMP_SVD,
121            )
122
123            # Transform gaussian map.
124            char_bounding_box = char_polygon.bounding_box
125
126            np_gaussian_map = cv.warpPerspective(
127                self.np_gaussian_map,
128                np_trans_mat,
129                (char_bounding_box.width, char_bounding_box.height),
130            )
131            score_map = ScoreMap(mat=np_gaussian_map, box=char_bounding_box)
132
133            # Fill score maps.
134            char_polygon.fill_score_map(score_map_max, score_map, keep_max_value=True)
135            char_polygon.fill_score_map(score_map_min, score_map, keep_min_value=True)
136
137        # Set the char overlapped area while preserving score gte threshold.
138        char_overlapped_mask = Mask.from_polygons(
139            shape,
140            char_polygons,
141            ElementSetOperationMode.INTERSECT,
142        )
143
144        preserving_score_min = self.init_config.gaussian_map_preserving_score_min
145        preserving_mask = Mask(mat=(score_map_max.mat >= preserving_score_min).astype(np.uint8))
146
147        neutralized_mask = Mask.from_masks(
148            shape,
149            [
150                char_overlapped_mask,
151                preserving_mask.to_inverted_mask(),
152            ],
153            ElementSetOperationMode.INTERSECT,
154        )
155
156        # Estimate the neutralized score.
157        np_delta: np.ndarray = score_map_max.mat - score_map_min.mat  # type: ignore
158        np_delta = np.clip(np_delta, 0.0, 1.0)
159        char_neutralized_score_map = ScoreMap(mat=np_delta)
160
161        neutralized_score_map = score_map_max.copy()
162        neutralized_mask.fill_score_map(neutralized_score_map, char_neutralized_score_map)
163
164        weight = self.init_config.weight_neutralized_score_map
165        score_map = ScoreMap(
166            mat=((1 - weight) * score_map_max.mat + weight * neutralized_score_map.mat)
167        )
168
169        debug = None
170        if run_config.enable_debug:
171            debug = CharHeatmapDefaultDebug(
172                score_map_max=score_map_max,
173                score_map_min=score_map_min,
174                char_overlapped_mask=char_overlapped_mask,
175                char_neutralized_score_map=char_neutralized_score_map,
176                neutralized_mask=neutralized_mask,
177                neutralized_score_map=neutralized_score_map,
178            )
179
180        return CharHeatmap(score_map=score_map, debug=debug)