vkit.engine.font.freetype

   1# Copyright 2022 vkit-x Administrator. All Rights Reserved.
   2#
   3# This project (vkit-x/vkit) is dual-licensed under commercial and SSPL licenses.
   4#
   5# The commercial license gives you the full rights to create and distribute software
   6# on your own terms without any SSPL license obligations. For more information,
   7# please see the "LICENSE_COMMERCIAL.txt" file.
   8#
   9# This project is also available under Server Side Public License (SSPL).
  10# The SSPL licensing is ideal for use cases such as open source projects with
  11# SSPL distribution, student/academic purposes, hobby projects, internal research
  12# projects without external distribution, or other projects where all SSPL
  13# obligations can be met. For more information, please see the "LICENSE_SSPL.txt" file.
  14from typing import Optional, Any, Callable, List, Sequence
  15import itertools
  16
  17import attrs
  18import numpy as np
  19from numpy.random import Generator as RandomGenerator
  20import cv2 as cv
  21import freetype
  22
  23from vkit.utility import sample_cv_resize_interpolation
  24from vkit.element import Image, Box, Mask, ScoreMap
  25from ..interface import (
  26    NoneTypeEngineInitConfig,
  27    NoneTypeEngineInitResource,
  28    Engine,
  29    EngineExecutorFactory,
  30)
  31from .type import (
  32    CharBox,
  33    FontEngineRunConfigGlyphSequence,
  34    FontEngineRunConfigStyle,
  35    FontEngineRunConfig,
  36    CharGlyph,
  37    TextLine,
  38)
  39
  40
  41def estimate_font_size(config: FontEngineRunConfig):
  42    style = config.style
  43
  44    # Estimate the font size based on height or width.
  45    if config.glyph_sequence == FontEngineRunConfigGlyphSequence.HORI_DEFAULT:
  46        font_size = round(config.height * style.font_size_ratio)
  47    elif config.glyph_sequence == FontEngineRunConfigGlyphSequence.VERT_DEFAULT:
  48        font_size = round(config.width * style.font_size_ratio)
  49    else:
  50        raise NotImplementedError()
  51
  52    font_size = int(np.clip(font_size, style.font_size_min, style.font_size_max))
  53    return font_size
  54
  55
  56def load_freetype_font_face(
  57    run_config: FontEngineRunConfig,
  58    lcd_compression_factor: Optional[int] = None,
  59):
  60    font_variant = run_config.font_variant
  61    if font_variant.is_ttc:
  62        assert font_variant.ttc_font_index is not None
  63        font_face = freetype.Face(str(font_variant.font_file), index=font_variant.ttc_font_index)
  64    else:
  65        font_face = freetype.Face(str(font_variant.font_file))
  66
  67    font_size = estimate_font_size(run_config)
  68
  69    # 1. "The nominal width, in 26.6 fractional points" and "The 26.6 fixed float format
  70    #    used to define fractional pixel coordinates. Here, 1 unit = 1/64 pixel", hence
  71    #    we need to multiple 64 here.
  72    # 2. `height` defaults to 0 and "If either the character width or
  73    #    height is zero, it is set equal to the other value",
  74    #    hence no need to set the `height` parameter.
  75    # 3. `vres` for vertical resolution, and defaults to 72.
  76    #    Since "pixel_size = point_size * resolution / 72", setting resolution to 72 equalize
  77    #    point_size and pixel_size. Since "If either the horizontal or vertical resolution is zero,
  78    #    it is set equal to the other value" and `vres` defaults to 72, we don't need to set the
  79    #    `vres` parameter.
  80    # 4. `hres` for horizontal resolution, and defaults to 72. If `lcd_compression_factor` is set,
  81    #    we need to cancel the sub-pixel rendering effect, by scaling up the same factor to `hres`.
  82    hres = 72
  83    if lcd_compression_factor is not None:
  84        hres *= lcd_compression_factor
  85    font_face.set_char_size(width=font_size * 64, hres=hres)
  86
  87    return font_face
  88
  89
  90def build_freetype_font_face_lcd_hc_matrix(lcd_compression_factor: int):
  91    # "hc" stands for "horizontal compression".
  92    return freetype.Matrix(
  93        int((1 / lcd_compression_factor) * 0x10000),
  94        int(0.0 * 0x10000),
  95        int(0.0 * 0x10000),
  96        int(1.0 * 0x10000),
  97    )
  98
  99
 100def trim_char_np_image_vert(np_image: np.ndarray):
 101    # (H, W) or (H, W, 3)
 102    np_vert_max: np.ndarray = np.amax(np_image, axis=1)
 103    if np_vert_max.ndim == 2:
 104        np_vert_max = np.amax(np_vert_max, axis=1)
 105    assert np_vert_max.ndim == 1
 106
 107    np_vert_nonzero = np.nonzero(np_vert_max)[0]
 108    if len(np_vert_nonzero) == 0:
 109        raise RuntimeError('trim_char_np_image_vert: empty np_image.')
 110
 111    up = int(np_vert_nonzero[0])
 112    down = int(np_vert_nonzero[-1])
 113
 114    height = np_image.shape[0]
 115    return np_image[up:down + 1], up, height - 1 - down
 116
 117
 118def trim_char_np_image_hori(np_image: np.ndarray):
 119    # (H, W) or (H, W, 3)
 120    np_hori_max: np.ndarray = np.amax(np_image, axis=0)
 121    if np_hori_max.ndim == 2:
 122        np_hori_max = np.amax(np_hori_max, axis=1)
 123    assert np_hori_max.ndim == 1
 124
 125    np_hori_nonzero = np.nonzero(np_hori_max)[0]
 126    if len(np_hori_nonzero) == 0:
 127        raise RuntimeError('trim_char_np_image_hori: empty np_image.')
 128
 129    left = int(np_hori_nonzero[0])
 130    right = int(np_hori_nonzero[-1])
 131
 132    width = np_image.shape[1]
 133    return np_image[:, left:right + 1], left, width - 1 - right
 134
 135
 136def build_char_glyph(
 137    config: FontEngineRunConfig,
 138    char: str,
 139    glyph: Any,
 140    np_image: np.ndarray,
 141):
 142    assert not char.isspace()
 143
 144    # References:
 145    # https://freetype.org/freetype2/docs/tutorial/step2.html
 146    # https://freetype-py.readthedocs.io/en/latest/glyph_slot.html
 147
 148    # "the distance from the baseline to the top-most glyph scanline"
 149    ascent = glyph.bitmap_top
 150
 151    # "It is always zero for horizontal layouts, and positive for vertical layouts."
 152    assert glyph.advance.y == 0
 153
 154    # Trim vertically to get pad_up & pad_down.
 155    np_image, pad_up, pad_down = trim_char_np_image_vert(np_image)
 156    ascent -= pad_up
 157
 158    # "bitmap’s left bearing"
 159    # NOTE: `bitmap_left` could be negative, we simply ignore the negative value.
 160    pad_left = max(0, glyph.bitmap_left)
 161
 162    # "It is always zero for horizontal layouts, and zero for vertical ones."
 163    assert glyph.advance.x > 0
 164    width = np_image.shape[1]
 165    # Aforementioned, 1 unit = 1/64 pixel.
 166    pad_right = round(glyph.advance.x / 64) - pad_left - width
 167    # Apply defensive clipping.
 168    pad_right = max(0, pad_right)
 169
 170    # Trim horizontally to increase pad_left & pad_right.
 171    np_image, pad_left_inc, pad_right_inc = trim_char_np_image_hori(np_image)
 172    pad_left += pad_left_inc
 173    pad_right += pad_right_inc
 174
 175    score_map = None
 176    if np_image.ndim == 2:
 177        # Apply gamma correction.
 178        np_alpha = np.power(
 179            np_image.astype(np.float32) / 255.0,
 180            config.style.glyph_color_gamma,
 181        )
 182        score_map = ScoreMap(mat=np_alpha)
 183
 184    # Reference statistics.
 185    font_variant = config.font_variant
 186    tag_to_font_glyph_info = font_variant.font_glyph_info_collection.tag_to_font_glyph_info
 187
 188    assert char in font_variant.char_to_tags
 189    tags = font_variant.char_to_tags[char]
 190
 191    font_glyph_info = None
 192    for tag in tags:
 193        assert tag in tag_to_font_glyph_info
 194        cur_font_glyph_info = tag_to_font_glyph_info[tag]
 195        if font_glyph_info is None:
 196            font_glyph_info = cur_font_glyph_info
 197        else:
 198            assert font_glyph_info == cur_font_glyph_info
 199
 200    assert font_glyph_info is not None
 201
 202    font_size = estimate_font_size(config)
 203    ref_ascent_plus_pad_up = round(
 204        font_glyph_info.ascent_plus_pad_up_min_to_font_size_ratio * font_size
 205    )
 206    ref_char_height = round(font_glyph_info.height_min_to_font_size_ratio * font_size)
 207    ref_char_width = round(font_glyph_info.width_min_to_font_size_ratio * font_size)
 208
 209    return CharGlyph(
 210        char=char,
 211        image=Image(mat=np_image),
 212        score_map=score_map,
 213        ascent=ascent,
 214        pad_up=pad_up,
 215        pad_down=pad_down,
 216        pad_left=pad_left,
 217        pad_right=pad_right,
 218        ref_ascent_plus_pad_up=ref_ascent_plus_pad_up,
 219        ref_char_height=ref_char_height,
 220        ref_char_width=ref_char_width,
 221    )
 222
 223
 224def render_char_glyphs_from_text(
 225    run_config: FontEngineRunConfig,
 226    font_face: freetype.Face,
 227    func_render_char_glyph: Callable[[FontEngineRunConfig, freetype.Face, str], CharGlyph],
 228    chars: Sequence[str],
 229):
 230    char_glyphs: List[CharGlyph] = []
 231    prev_num_spaces_for_char_glyphs: List[int] = []
 232    num_spaces = 0
 233    for idx, char in enumerate(chars):
 234        if char.isspace():
 235            num_spaces += 1
 236            continue
 237
 238        char_glyphs.append(func_render_char_glyph(run_config, font_face, char))
 239
 240        if idx == 0 and num_spaces > 0:
 241            raise RuntimeError('Leading space(s) detected.')
 242        prev_num_spaces_for_char_glyphs.append(num_spaces)
 243        num_spaces = 0
 244
 245    if num_spaces > 0:
 246        raise RuntimeError('Trailing space(s) detected.')
 247
 248    return char_glyphs, prev_num_spaces_for_char_glyphs
 249
 250
 251def get_kerning_limits_hori_default(
 252    char_glyphs: Sequence[CharGlyph],
 253    prev_num_spaces_for_char_glyphs: Sequence[int],
 254):
 255    assert char_glyphs
 256    ascent_max = max(char_glyph.ascent for char_glyph in char_glyphs)
 257
 258    kerning_limits: List[int] = []
 259
 260    prev_glyph_mask = None
 261    prev_np_glyph_mask = None
 262    prev_glyph_mask_up = None
 263    prev_glyph_mask_down = None
 264    for char_glyph, prev_num_spaces in zip(char_glyphs, prev_num_spaces_for_char_glyphs):
 265        glyph_mask = char_glyph.get_glyph_mask()
 266        np_glyph_mask = glyph_mask.mat
 267        glyph_mask_up = ascent_max - char_glyph.ascent
 268        glyph_mask_down = glyph_mask_up + np_glyph_mask.shape[0] - 1
 269
 270        if prev_num_spaces == 0 and prev_np_glyph_mask is not None:
 271            assert prev_glyph_mask is not None
 272            assert prev_glyph_mask_up is not None
 273            assert prev_glyph_mask_down is not None
 274
 275            overlap_up = max(prev_glyph_mask_up, glyph_mask_up)
 276            overlap_down = min(prev_glyph_mask_down, glyph_mask_down)
 277            if overlap_up <= overlap_down:
 278                overlap_prev_np_glyph_mask = \
 279                    prev_np_glyph_mask[overlap_up - prev_glyph_mask_up:
 280                                       overlap_down - prev_glyph_mask_up + 1]
 281                overlap_np_glyph_mask = \
 282                    np_glyph_mask[overlap_up - glyph_mask_up:
 283                                  overlap_down - glyph_mask_up + 1]
 284
 285                kerning_limit = 1
 286                while kerning_limit < prev_glyph_mask.width / 2 \
 287                        and kerning_limit < glyph_mask.width / 2:
 288                    prev_np_glyph_mask_tail = overlap_prev_np_glyph_mask[:, -kerning_limit:]
 289                    np_glyph_mask_head = overlap_np_glyph_mask[:, :kerning_limit]
 290                    if (prev_np_glyph_mask_tail & np_glyph_mask_head).any():
 291                        # Intersection detected.
 292                        kerning_limit -= 1
 293                        break
 294                    kerning_limit += 1
 295
 296                kerning_limits.append(kerning_limit)
 297
 298            else:
 299                # Not overlapped.
 300                kerning_limits.append(0)
 301
 302        else:
 303            # Isolated by word space or being the first glyph, skip.
 304            kerning_limits.append(0)
 305
 306        prev_glyph_mask = glyph_mask
 307        prev_np_glyph_mask = np_glyph_mask
 308        prev_glyph_mask_up = glyph_mask_up
 309        prev_glyph_mask_down = glyph_mask_down
 310
 311    return kerning_limits
 312
 313
 314def render_char_glyphs_in_text_line(
 315    style: FontEngineRunConfigStyle,
 316    text_line_height: int,
 317    text_line_width: int,
 318    char_glyphs: Sequence[CharGlyph],
 319    char_boxes: Sequence[CharBox],
 320):
 321    # Genrate text line image.
 322    np_image = np.full((text_line_height, text_line_width, 3), 255, dtype=np.uint8)
 323    np_mask = np.zeros((text_line_height, text_line_width), dtype=np.uint8)
 324    score_map = None
 325
 326    if char_glyphs[0].image.mat.ndim == 2:
 327        score_map = ScoreMap.from_shape((text_line_height, text_line_width))
 328
 329        # Default or monochrome.
 330        for char_glyph, char_box in zip(char_glyphs, char_boxes):
 331            assert char_glyph.score_map
 332
 333            char_glyph_mask = char_glyph.get_glyph_mask(box=char_box.box)
 334
 335            # Fill color based on RGBA & alpha.
 336            np_char_image = np.full(
 337                (char_glyph.height, char_glyph.width, 4),
 338                (*style.glyph_color, 0),
 339                dtype=np.uint8,
 340            )
 341            np_char_image[:, :, 3] = (char_glyph.score_map.mat * 255).astype(np.uint8)
 342
 343            # To RGB.
 344            np_char_image = cv.cvtColor(np_char_image, cv.COLOR_RGBA2RGB)
 345
 346            # Paste to text line.
 347            char_glyph_mask.fill_np_array(np_image, np_char_image)
 348            char_glyph_mask.fill_np_array(np_mask, 1)
 349            char_box.box.fill_score_map(
 350                score_map,
 351                char_glyph.score_map,
 352                keep_max_value=True,
 353            )
 354
 355    elif char_glyphs[0].image.mat.ndim == 3:
 356        # LCD.
 357        for char_glyph, char_box in zip(char_glyphs, char_boxes):
 358            char_glyph_mask = char_glyph.get_glyph_mask(box=char_box.box)
 359
 360            # NOTE: the `glyph_color` option is ignored in LCD mode.
 361            # Gamma correction.
 362            np_char_image = np.power(
 363                char_glyph.image.mat / 255.0,
 364                style.glyph_color_gamma,
 365            )
 366            np_char_image = ((1 - np_char_image) * 255).astype(np.uint8)  # type: ignore
 367
 368            # Paste to text line.
 369            char_glyph_mask.fill_np_array(np_image, np_char_image)
 370            char_glyph_mask.fill_np_array(np_mask, 1)
 371
 372    else:
 373        raise NotImplementedError()
 374
 375    return (
 376        Image(mat=np_image),
 377        Mask(mat=np_mask),
 378        score_map,
 379        char_boxes,
 380    )
 381
 382
 383def place_char_glyphs_in_text_line_hori_default(
 384    run_config: FontEngineRunConfig,
 385    char_glyphs: Sequence[CharGlyph],
 386    prev_num_spaces_for_char_glyphs: Sequence[int],
 387    kerning_limits: Sequence[int],
 388    rng: RandomGenerator,
 389):
 390    style = run_config.style
 391
 392    assert char_glyphs
 393    char_widths_avg = np.mean([char_glyph.width for char_glyph in char_glyphs])
 394
 395    char_space_min = char_widths_avg * style.char_space_min
 396    char_space_max = char_widths_avg * style.char_space_max
 397    char_space_mean = char_widths_avg * style.char_space_mean
 398    char_space_std = char_widths_avg * style.char_space_std
 399
 400    word_space_min = char_widths_avg * style.word_space_min
 401    word_space_max = char_widths_avg * style.word_space_max
 402    word_space_mean = char_widths_avg * style.word_space_mean
 403    word_space_std = char_widths_avg * style.word_space_std
 404
 405    ascent_plus_pad_up_max = max(
 406        itertools.chain.from_iterable(
 407            (char_glyph.ascent + char_glyph.pad_up, char_glyph.ref_ascent_plus_pad_up)
 408            for char_glyph in char_glyphs
 409        )
 410    )
 411
 412    text_line_height = max(char_glyph.ref_char_height for char_glyph in char_glyphs)
 413
 414    char_boxes: List[CharBox] = []
 415    hori_offset = 0
 416    for char_idx, (char_glyph, prev_num_spaces, kerning_limit) in enumerate(
 417        zip(
 418            char_glyphs,
 419            prev_num_spaces_for_char_glyphs,
 420            kerning_limits,
 421        )
 422    ):
 423        # "Stick" chars together.
 424        hori_offset -= kerning_limit
 425
 426        # Shift by space.
 427        if prev_num_spaces > 0:
 428            # Random word space(s).
 429            space = 0
 430            for _ in range(prev_num_spaces):
 431                space += round(
 432                    np.clip(
 433                        rng.normal(loc=word_space_mean, scale=word_space_std),
 434                        word_space_min,
 435                        word_space_max,
 436                    )  # type: ignore
 437                )
 438
 439        else:
 440            # Random char space.
 441            if rng.random() < style.prob_set_char_space_min:
 442                space = round(char_space_min)
 443            else:
 444                space = round(
 445                    np.clip(
 446                        rng.normal(loc=char_space_mean, scale=char_space_std),
 447                        char_space_min,
 448                        char_space_max,
 449                    )  # type: ignore
 450                )
 451
 452        hori_offset += space
 453
 454        # Place char box.
 455        up = ascent_plus_pad_up_max - char_glyph.ascent
 456        down = up + char_glyph.height - 1
 457
 458        left = hori_offset + char_glyph.pad_left
 459        if char_idx == 0:
 460            # Ignore the leading padding.
 461            left = 0
 462        right = left + char_glyph.width - 1
 463
 464        assert not char_glyph.char.isspace()
 465        char_boxes.append(
 466            CharBox(
 467                char=char_glyph.char,
 468                box=Box(
 469                    up=up,
 470                    down=down,
 471                    left=left,
 472                    right=right,
 473                ),
 474            )
 475        )
 476
 477        # Update the height of text line.
 478        text_line_height = max(text_line_height, down + 1 + char_glyph.pad_down)
 479
 480        # Move offset.
 481        hori_offset = right + 1
 482        if char_idx < len(char_glyphs) - 1:
 483            hori_offset += char_glyph.pad_right
 484
 485    text_line_width = hori_offset
 486
 487    return render_char_glyphs_in_text_line(
 488        style=style,
 489        text_line_height=text_line_height,
 490        text_line_width=text_line_width,
 491        char_glyphs=char_glyphs,
 492        char_boxes=char_boxes,
 493    )
 494
 495
 496def place_char_glyphs_in_text_line_vert_default(
 497    run_config: FontEngineRunConfig,
 498    char_glyphs: Sequence[CharGlyph],
 499    prev_num_spaces_for_char_glyphs: Sequence[int],
 500    rng: RandomGenerator,
 501):
 502    style = run_config.style
 503
 504    assert char_glyphs
 505    char_widths_avg = np.mean([char_glyph.width for char_glyph in char_glyphs])
 506
 507    char_space_min = char_widths_avg * style.char_space_min
 508    char_space_max = char_widths_avg * style.char_space_max
 509    char_space_mean = char_widths_avg * style.char_space_mean
 510    char_space_std = char_widths_avg * style.char_space_std
 511
 512    word_space_min = char_widths_avg * style.word_space_min
 513    word_space_max = char_widths_avg * style.word_space_max
 514    word_space_mean = char_widths_avg * style.word_space_mean
 515    word_space_std = char_widths_avg * style.word_space_std
 516
 517    text_line_width = max(
 518        itertools.chain.from_iterable((
 519            char_glyph.pad_left + char_glyph.width + char_glyph.pad_right,
 520            char_glyph.ref_char_width,
 521        ) for char_glyph in char_glyphs)
 522    )
 523
 524    text_line_width_mid = text_line_width // 2
 525
 526    char_boxes: List[CharBox] = []
 527    vert_offset = 0
 528    for char_idx, (char_glyph, prev_num_spaces) in enumerate(
 529        zip(char_glyphs, prev_num_spaces_for_char_glyphs)
 530    ):
 531        # Shift by space.
 532        if prev_num_spaces > 0:
 533            # Random word space(s).
 534            space = 0
 535            for _ in range(prev_num_spaces):
 536                space += round(
 537                    np.clip(
 538                        rng.normal(loc=word_space_mean, scale=word_space_std),
 539                        word_space_min,
 540                        word_space_max,
 541                    )  # type: ignore
 542                )
 543
 544        else:
 545            # Random char space.
 546            if rng.random() < style.prob_set_char_space_min:
 547                space = round(char_space_min)
 548            else:
 549                space = round(
 550                    np.clip(
 551                        rng.normal(loc=char_space_mean, scale=char_space_std),
 552                        char_space_min,
 553                        char_space_max,
 554                    )  # type: ignore
 555                )
 556
 557        vert_offset += space
 558
 559        # Place char box.
 560        up = vert_offset + char_glyph.pad_up
 561        if char_idx == 0:
 562            # Ignore the leading padding.
 563            up = 0
 564
 565        down = up + char_glyph.height - 1
 566
 567        # Vertical align in middle.
 568        left = text_line_width_mid - char_glyph.width // 2
 569        right = left + char_glyph.width - 1
 570
 571        assert not char_glyph.char.isspace()
 572        char_boxes.append(
 573            CharBox(
 574                char=char_glyph.char,
 575                box=Box(
 576                    up=up,
 577                    down=down,
 578                    left=left,
 579                    right=right,
 580                ),
 581            )
 582        )
 583
 584        # Move offset.
 585        vert_offset = down + 1
 586        if char_idx < len(char_glyphs) - 1:
 587            vert_offset += char_glyph.pad_down
 588
 589    text_line_height = vert_offset
 590
 591    return render_char_glyphs_in_text_line(
 592        style=style,
 593        text_line_height=text_line_height,
 594        text_line_width=text_line_width,
 595        char_glyphs=char_glyphs,
 596        char_boxes=char_boxes,
 597    )
 598
 599
 600def resize_and_trim_text_line_hori_default(
 601    run_config: FontEngineRunConfig,
 602    cv_resize_interpolation_enlarge: int,
 603    cv_resize_interpolation_shrink: int,
 604    image: Image,
 605    mask: Mask,
 606    score_map: Optional[ScoreMap],
 607    char_boxes: Sequence[CharBox],
 608    char_glyphs: Sequence[CharGlyph],
 609):
 610    # Resize if image height too small or too large.
 611    is_too_small = (image.height / run_config.height < 0.8)
 612    is_too_large = (image.height > run_config.height)
 613
 614    cv_resize_interpolation = cv_resize_interpolation_enlarge
 615    if is_too_large:
 616        cv_resize_interpolation = cv_resize_interpolation_shrink
 617
 618    if is_too_small or is_too_large:
 619        resized_image = image.to_resized_image(
 620            resized_height=run_config.height,
 621            cv_resize_interpolation=cv_resize_interpolation,
 622        )
 623        resized_mask = mask.to_resized_mask(
 624            resized_height=run_config.height,
 625            cv_resize_interpolation=cv_resize_interpolation,
 626        )
 627        resized_char_boxes = [
 628            char_box.to_conducted_resized_char_box(
 629                shapable_or_shape=image,
 630                resized_height=run_config.height,
 631            ) for char_box in char_boxes
 632        ]
 633
 634        image = resized_image
 635        mask = resized_mask
 636        char_boxes = resized_char_boxes
 637
 638        if score_map:
 639            score_map = score_map.to_resized_score_map(
 640                resized_height=run_config.height,
 641                cv_resize_interpolation=cv_resize_interpolation,
 642            )
 643
 644    # Pad vertically.
 645    if image.height != run_config.height:
 646        pad_vert = run_config.height - image.height
 647        assert pad_vert > 0
 648        pad_up = pad_vert // 2
 649        pad_down = pad_vert - pad_up
 650
 651        np_image = np.full((run_config.height, image.width, 3), 255, dtype=np.uint8)
 652        np_image[pad_up:-pad_down] = image.mat
 653        image.assign_mat(np_image)
 654
 655        np_mask = np.zeros((run_config.height, image.width), dtype=np.uint8)
 656        np_mask[pad_up:-pad_down] = mask.mat
 657        mask.assign_mat(np_mask)
 658
 659        padded_char_boxes = []
 660        for char_box in char_boxes:
 661            box = attrs.evolve(
 662                char_box.box,
 663                up=char_box.up + pad_up,
 664                down=char_box.down + pad_up,
 665            )
 666            padded_char_boxes.append(attrs.evolve(char_box, box=box))
 667        char_boxes = padded_char_boxes
 668
 669        if score_map:
 670            padded_score_map = ScoreMap.from_shape((run_config.height, image.width))
 671            with padded_score_map.writable_context:
 672                padded_score_map.mat[pad_up:-pad_down] = score_map.mat
 673            score_map = padded_score_map
 674
 675    # Trim.
 676    if image.width > run_config.width:
 677        last_char_box_idx = len(char_boxes) - 1
 678        while last_char_box_idx >= 0 and char_boxes[last_char_box_idx].right >= run_config.width:
 679            last_char_box_idx -= 1
 680
 681        if last_char_box_idx == len(char_boxes) - 1:
 682            # Corner case: char_boxes[-1].right < config.width but mage.width > config.width.
 683            # This is caused by glyph padding. The solution is to drop this char.
 684            last_char_box_idx -= 1
 685
 686        if last_char_box_idx < 0 or char_boxes[last_char_box_idx].right >= run_config.width:
 687            # Cannot trim.
 688            return None, None, None, None, -1
 689
 690        last_char_box = char_boxes[last_char_box_idx]
 691        last_char_box_right = last_char_box.right
 692
 693        # Clean up residual pixels.
 694        first_trimed_char_box = char_boxes[last_char_box_idx + 1]
 695        if first_trimed_char_box.left <= last_char_box_right:
 696            first_trimed_char_glyph = char_glyphs[last_char_box_idx + 1]
 697
 698            first_trimed_char_glyph_mask = first_trimed_char_glyph.get_glyph_mask(
 699                box=first_trimed_char_box.box,
 700                enable_resize=True,
 701                cv_resize_interpolation=cv_resize_interpolation,
 702            )
 703            first_trimed_char_glyph_mask.fill_image(image, (255, 255, 255))
 704            first_trimed_char_glyph_mask.fill_mask(mask, 0)
 705
 706            if first_trimed_char_glyph.score_map:
 707                assert score_map
 708
 709                first_trimed_char_score_map = first_trimed_char_glyph.score_map
 710                if first_trimed_char_score_map.shape != first_trimed_char_box.shape:
 711                    first_trimed_char_score_map = first_trimed_char_score_map.to_resized_score_map(
 712                        resized_height=first_trimed_char_box.height,
 713                        resized_width=first_trimed_char_box.width,
 714                        cv_resize_interpolation=cv_resize_interpolation,
 715                    )
 716
 717                last_char_score_map = char_glyphs[last_char_box_idx].score_map
 718                assert last_char_score_map
 719                if last_char_score_map.shape != last_char_box.shape:
 720                    last_char_score_map = last_char_score_map.to_resized_score_map(
 721                        resized_height=last_char_box.height,
 722                        resized_width=last_char_box.width,
 723                        cv_resize_interpolation=cv_resize_interpolation,
 724                    )
 725
 726                first_trimed_char_box.box.fill_score_map(score_map, 0)
 727                last_char_box.box.fill_score_map(
 728                    score_map,
 729                    last_char_score_map,
 730                    keep_max_value=True,
 731                )
 732
 733        char_boxes = char_boxes[:last_char_box_idx + 1]
 734        image.assign_mat(image.mat[:, :last_char_box_right + 1])
 735        mask.assign_mat(mask.mat[:, :last_char_box_right + 1])
 736
 737        if score_map:
 738            score_map.assign_mat(score_map.mat[:, :last_char_box_right + 1])
 739
 740    return image, mask, score_map, char_boxes, cv_resize_interpolation
 741
 742
 743def resize_and_trim_text_line_vert_default(
 744    run_config: FontEngineRunConfig,
 745    cv_resize_interpolation_enlarge: int,
 746    cv_resize_interpolation_shrink: int,
 747    image: Image,
 748    mask: Mask,
 749    score_map: Optional[ScoreMap],
 750    char_boxes: Sequence[CharBox],
 751):
 752    # Resize if image width too small or too large.
 753    is_too_small = (image.width / run_config.width < 0.8)
 754    is_too_large = (image.width > run_config.width)
 755
 756    cv_resize_interpolation = cv_resize_interpolation_enlarge
 757    if is_too_large:
 758        cv_resize_interpolation = cv_resize_interpolation_shrink
 759
 760    if is_too_small or is_too_large:
 761        resized_image = image.to_resized_image(
 762            resized_width=run_config.width,
 763            cv_resize_interpolation=cv_resize_interpolation,
 764        )
 765        resized_mask = mask.to_resized_mask(
 766            resized_width=run_config.width,
 767            cv_resize_interpolation=cv_resize_interpolation,
 768        )
 769        resized_char_boxes = [
 770            char_box.to_conducted_resized_char_box(
 771                shapable_or_shape=image,
 772                resized_width=run_config.width,
 773            ) for char_box in char_boxes
 774        ]
 775
 776        image = resized_image
 777        mask = resized_mask
 778        char_boxes = resized_char_boxes
 779
 780        if score_map:
 781            score_map = score_map.to_resized_score_map(
 782                resized_width=run_config.width,
 783                cv_resize_interpolation=cv_resize_interpolation,
 784            )
 785
 786    # Pad horizontally.
 787    if image.width != run_config.width:
 788        pad_hori = run_config.width - image.width
 789        assert pad_hori > 0
 790        pad_left = pad_hori // 2
 791        pad_right = pad_hori - pad_left
 792
 793        np_image = np.full((image.height, run_config.width, 3), 255, dtype=np.uint8)
 794        np_image[:, pad_left:-pad_right] = image.mat
 795        image.assign_mat(np_image)
 796
 797        np_mask = np.zeros((image.height, run_config.width), dtype=np.uint8)
 798        np_mask[:, pad_left:-pad_right] = mask.mat
 799        mask.assign_mat(np_mask)
 800
 801        padded_char_boxes = []
 802        for char_box in char_boxes:
 803            box = attrs.evolve(
 804                char_box.box,
 805                left=char_box.left + pad_left,
 806                right=char_box.right + pad_right,
 807            )
 808            padded_char_boxes.append(attrs.evolve(char_box, box=box))
 809        char_boxes = padded_char_boxes
 810
 811        if score_map:
 812            padded_score_map = ScoreMap.from_shape((image.height, run_config.width))
 813            padded_score_map.mat[:, pad_left:-pad_right] = score_map.mat
 814            score_map = padded_score_map
 815
 816    # Trim.
 817    if image.height > run_config.height:
 818        last_char_box_idx = len(char_boxes) - 1
 819        while last_char_box_idx >= 0 and char_boxes[last_char_box_idx].down >= run_config.height:
 820            last_char_box_idx -= 1
 821
 822        if last_char_box_idx == len(char_boxes) - 1:
 823            last_char_box_idx -= 1
 824
 825        if last_char_box_idx < 0 or char_boxes[last_char_box_idx].down >= run_config.height:
 826            # Cannot trim.
 827            return None, None, None, None, -1
 828
 829        last_char_box_down = char_boxes[last_char_box_idx].down
 830        char_boxes = char_boxes[:last_char_box_idx + 1]
 831        image.assign_mat(image.mat[:last_char_box_down + 1])
 832        mask.assign_mat(mask.mat[:last_char_box_down + 1])
 833
 834        if score_map:
 835            score_map.assign_mat(score_map.mat[:last_char_box_down + 1])
 836
 837    return image, mask, score_map, char_boxes, cv_resize_interpolation
 838
 839
 840def render_text_line_meta(
 841    run_config: FontEngineRunConfig,
 842    font_face: freetype.Face,
 843    func_render_char_glyph: Callable[[FontEngineRunConfig, freetype.Face, str], CharGlyph],
 844    rng: RandomGenerator,
 845    cv_resize_interpolation_enlarge: int = cv.INTER_CUBIC,
 846    cv_resize_interpolation_shrink: int = cv.INTER_AREA,
 847):
 848    (
 849        char_glyphs,
 850        prev_num_spaces_for_char_glyphs,
 851    ) = render_char_glyphs_from_text(
 852        run_config=run_config,
 853        font_face=font_face,
 854        func_render_char_glyph=func_render_char_glyph,
 855        chars=run_config.chars,
 856    )
 857    if not char_glyphs:
 858        return None
 859
 860    if run_config.glyph_sequence == FontEngineRunConfigGlyphSequence.HORI_DEFAULT:
 861        kerning_limits = get_kerning_limits_hori_default(
 862            char_glyphs,
 863            prev_num_spaces_for_char_glyphs,
 864        )
 865        (
 866            image,
 867            mask,
 868            score_map,
 869            char_boxes,
 870        ) = place_char_glyphs_in_text_line_hori_default(
 871            run_config=run_config,
 872            char_glyphs=char_glyphs,
 873            prev_num_spaces_for_char_glyphs=prev_num_spaces_for_char_glyphs,
 874            kerning_limits=kerning_limits,
 875            rng=rng,
 876        )
 877        (
 878            image,
 879            mask,
 880            score_map,
 881            char_boxes,
 882            cv_resize_interpolation,
 883        ) = resize_and_trim_text_line_hori_default(
 884            run_config=run_config,
 885            cv_resize_interpolation_enlarge=cv_resize_interpolation_enlarge,
 886            cv_resize_interpolation_shrink=cv_resize_interpolation_shrink,
 887            image=image,
 888            mask=mask,
 889            score_map=score_map,
 890            char_boxes=char_boxes,
 891            char_glyphs=char_glyphs,
 892        )
 893        is_hori = True
 894
 895    elif run_config.glyph_sequence == FontEngineRunConfigGlyphSequence.VERT_DEFAULT:
 896        # NOTE: No kerning limit detection for VERT_DEFAULT mode.
 897        (
 898            image,
 899            mask,
 900            score_map,
 901            char_boxes,
 902        ) = place_char_glyphs_in_text_line_vert_default(
 903            run_config=run_config,
 904            char_glyphs=char_glyphs,
 905            prev_num_spaces_for_char_glyphs=prev_num_spaces_for_char_glyphs,
 906            rng=rng,
 907        )
 908        (
 909            image,
 910            mask,
 911            score_map,
 912            char_boxes,
 913            cv_resize_interpolation,
 914        ) = resize_and_trim_text_line_vert_default(
 915            run_config=run_config,
 916            cv_resize_interpolation_enlarge=cv_resize_interpolation_enlarge,
 917            cv_resize_interpolation_shrink=cv_resize_interpolation_shrink,
 918            image=image,
 919            mask=mask,
 920            score_map=score_map,
 921            char_boxes=char_boxes,
 922        )
 923        is_hori = False
 924
 925    else:
 926        raise NotImplementedError()
 927
 928    if image is None:
 929        return None
 930    else:
 931        assert mask is not None
 932        assert char_boxes is not None
 933
 934        char_idx = 0
 935        non_space_count = 0
 936        while char_idx < len(run_config.chars) and non_space_count < len(char_boxes):
 937            if not run_config.chars[char_idx].isspace():
 938                non_space_count += 1
 939            char_idx += 1
 940        assert non_space_count == len(char_boxes)
 941
 942        box = Box.from_shapable(image)
 943        image = image.to_box_attached(box)
 944        mask = mask.to_box_attached(box)
 945        if score_map:
 946            score_map = score_map.to_box_attached(box)
 947
 948        return TextLine(
 949            image=image,
 950            mask=mask,
 951            score_map=score_map,
 952            char_boxes=char_boxes,
 953            char_glyphs=char_glyphs[:len(char_boxes)],
 954            cv_resize_interpolation=cv_resize_interpolation,
 955            font_size=estimate_font_size(run_config),
 956            style=run_config.style,
 957            text=''.join(run_config.chars[:char_idx]),
 958            is_hori=is_hori,
 959            font_variant=run_config.font_variant if run_config.return_font_variant else None,
 960        )
 961
 962
 963class FontFreetypeDefaultEngine(
 964    Engine[
 965        NoneTypeEngineInitConfig,
 966        NoneTypeEngineInitResource,
 967        FontEngineRunConfig,
 968        Optional[TextLine],
 969    ]
 970):  # yapf: disable
 971
 972    @classmethod
 973    def get_type_name(cls) -> str:
 974        return 'freetype_default'
 975
 976    @classmethod
 977    def render_char_glyph(
 978        cls,
 979        run_config: FontEngineRunConfig,
 980        font_face: freetype.Face,
 981        char: str,
 982    ):
 983        load_char_flags = freetype.FT_LOAD_RENDER  # type: ignore
 984        if run_config.style.freetype_force_autohint:
 985            load_char_flags |= freetype.FT_LOAD_FORCE_AUTOHINT  # type: ignore
 986        font_face.load_char(char, load_char_flags)
 987
 988        glyph = font_face.glyph
 989        bitmap = glyph.bitmap
 990
 991        height = bitmap.rows
 992        width = bitmap.width
 993        assert width == bitmap.pitch
 994
 995        # (H, W), [0, 255]
 996        np_image = np.asarray(bitmap.buffer, dtype=np.uint8).reshape(height, width)
 997
 998        return build_char_glyph(run_config, char, glyph, np_image)
 999
1000    def run(
1001        self,
1002        run_config: FontEngineRunConfig,
1003        rng: Optional[RandomGenerator] = None,
1004    ) -> Optional[TextLine]:
1005        assert rng is not None
1006
1007        font_face = load_freetype_font_face(run_config)
1008        return render_text_line_meta(
1009            run_config=run_config,
1010            font_face=font_face,
1011            func_render_char_glyph=self.render_char_glyph,
1012            rng=rng,
1013            cv_resize_interpolation_enlarge=sample_cv_resize_interpolation(rng),
1014            cv_resize_interpolation_shrink=sample_cv_resize_interpolation(
1015                rng,
1016                include_cv_inter_area=True,
1017            ),
1018        )
1019
1020
1021font_freetype_default_engine_executor_factory = EngineExecutorFactory(FontFreetypeDefaultEngine)
1022
1023
1024class FontFreetypeLcdEngine(
1025    Engine[
1026        NoneTypeEngineInitConfig,
1027        NoneTypeEngineInitResource,
1028        FontEngineRunConfig,
1029        Optional[TextLine],
1030    ]
1031):  # yapf: disable
1032
1033    @classmethod
1034    def get_type_name(cls) -> str:
1035        return 'freetype_lcd'
1036
1037    @classmethod
1038    def render_char_glyph(
1039        cls,
1040        run_config: FontEngineRunConfig,
1041        font_face: freetype.Face,
1042        lcd_hc_matrix: freetype.Matrix,
1043        char: str,
1044    ):
1045        load_char_flags = freetype.FT_LOAD_RENDER | freetype.FT_LOAD_TARGET_LCD  # type: ignore
1046        if run_config.style.freetype_force_autohint:
1047            load_char_flags |= freetype.FT_LOAD_FORCE_AUTOHINT  # type: ignore
1048        font_face.set_transform(lcd_hc_matrix, freetype.Vector(0, 0))
1049        font_face.load_char(char, load_char_flags)
1050
1051        glyph = font_face.glyph
1052        bitmap = glyph.bitmap
1053
1054        height = bitmap.rows
1055        pitch = bitmap.pitch
1056        flatten_width = bitmap.width
1057        width = flatten_width // 3
1058
1059        # (H, W, 3), [0, 255]
1060        np_image = np.asarray(bitmap.buffer, dtype=np.uint8).reshape(height, pitch)
1061        np_image = np_image[:, :width * 3].reshape(height, width, 3)
1062
1063        return build_char_glyph(run_config, char, glyph, np_image)
1064
1065    @classmethod
1066    def bind_render_char_glyph(cls, lcd_hc_matrix: freetype.Matrix):
1067        return lambda config, font_face, char: cls.render_char_glyph(
1068            config,
1069            font_face,
1070            lcd_hc_matrix,
1071            char,
1072        )
1073
1074    def run(
1075        self,
1076        run_config: FontEngineRunConfig,
1077        rng: Optional[RandomGenerator] = None,
1078    ) -> Optional[TextLine]:
1079        assert rng is not None
1080
1081        lcd_compression_factor = 10
1082        font_face = load_freetype_font_face(
1083            run_config,
1084            lcd_compression_factor=lcd_compression_factor,
1085        )
1086        lcd_hc_matrix = build_freetype_font_face_lcd_hc_matrix(lcd_compression_factor)
1087        return render_text_line_meta(
1088            run_config=run_config,
1089            font_face=font_face,
1090            func_render_char_glyph=self.bind_render_char_glyph(lcd_hc_matrix),
1091            rng=rng,
1092            cv_resize_interpolation_enlarge=sample_cv_resize_interpolation(rng),
1093            cv_resize_interpolation_shrink=sample_cv_resize_interpolation(
1094                rng,
1095                include_cv_inter_area=True,
1096            ),
1097        )
1098
1099
1100font_freetype_lcd_engine_executor_factory = EngineExecutorFactory(FontFreetypeLcdEngine)
1101
1102
1103class FontFreetypeMonochromeEngine(
1104    Engine[
1105        NoneTypeEngineInitConfig,
1106        NoneTypeEngineInitResource,
1107        FontEngineRunConfig,
1108        Optional[TextLine],
1109    ]
1110):  # yapf: disable
1111
1112    @classmethod
1113    def get_type_name(cls) -> str:
1114        return 'freetype_monochrome'
1115
1116    @classmethod
1117    def render_char_glyph(
1118        cls,
1119        run_config: FontEngineRunConfig,
1120        font_face: freetype.Face,
1121        char: str,
1122    ):
1123        load_char_flags = freetype.FT_LOAD_RENDER | freetype.FT_LOAD_TARGET_MONO  # type: ignore
1124        if run_config.style.freetype_force_autohint:
1125            load_char_flags |= freetype.FT_LOAD_FORCE_AUTOHINT  # type: ignore
1126        font_face.load_char(char, load_char_flags)
1127
1128        glyph = font_face.glyph
1129        bitmap = glyph.bitmap
1130
1131        height = bitmap.rows
1132        width = bitmap.width
1133        pitch = bitmap.pitch
1134
1135        # Performance optimization.
1136        bitmap_buffer = bitmap.buffer
1137
1138        data = []
1139        for row_idx in range(height):
1140            row = []
1141            for pitch_idx in range(pitch):
1142                byte = bitmap_buffer[row_idx * pitch + pitch_idx]
1143                bits = []
1144                for _ in range(8):
1145                    bits.append(int((byte & 1) == 1))
1146                    byte = byte >> 1
1147                row.extend(bit * 255 for bit in reversed(bits))
1148            data.append(row[:width])
1149
1150        np_image = np.asarray(data, dtype=np.uint8)
1151        assert np_image.shape == (height, width)
1152
1153        return build_char_glyph(run_config, char, glyph, np_image)
1154
1155    def run(
1156        self,
1157        run_config: FontEngineRunConfig,
1158        rng: Optional[RandomGenerator] = None,
1159    ) -> Optional[TextLine]:
1160        assert rng is not None
1161
1162        font_face = load_freetype_font_face(run_config)
1163        return render_text_line_meta(
1164            run_config=run_config,
1165            font_face=font_face,
1166            func_render_char_glyph=self.render_char_glyph,
1167            rng=rng,
1168            cv_resize_interpolation_enlarge=cv.INTER_NEAREST_EXACT,
1169            cv_resize_interpolation_shrink=cv.INTER_NEAREST_EXACT,
1170        )
1171
1172
1173font_freetype_monochrome_engine_executor_factory = EngineExecutorFactory(
1174    FontFreetypeMonochromeEngine
1175)