vkit.engine.font.freetype

   1# Copyright 2022 vkit-x Administrator. All Rights Reserved.
   2#
   3# This project (vkit-x/vkit) is dual-licensed under commercial and SSPL licenses.
   4#
   5# The commercial license gives you the full rights to create and distribute software
   6# on your own terms without any SSPL license obligations. For more information,
   7# please see the "LICENSE_COMMERCIAL.txt" file.
   8#
   9# This project is also available under Server Side Public License (SSPL).
  10# The SSPL licensing is ideal for use cases such as open source projects with
  11# SSPL distribution, student/academic purposes, hobby projects, internal research
  12# projects without external distribution, or other projects where all SSPL
  13# obligations can be met. For more information, please see the "LICENSE_SSPL.txt" file.
  14from typing import Optional, Any, Callable, List, Sequence
  15import itertools
  16
  17import attrs
  18import numpy as np
  19from numpy.random import Generator as RandomGenerator
  20import cv2 as cv
  21import freetype
  22
  23from vkit.utility import sample_cv_resize_interpolation
  24from vkit.element import Image, Box, Mask, ScoreMap
  25from vkit.engine.interface import (
  26    NoneTypeEngineInitConfig,
  27    NoneTypeEngineInitResource,
  28    Engine,
  29    EngineExecutorFactory,
  30)
  31from .type import (
  32    CharBox,
  33    FontEngineRunConfigGlyphSequence,
  34    FontEngineRunConfigStyle,
  35    FontEngineRunConfig,
  36    CharGlyph,
  37    TextLine,
  38)
  39
  40
  41def estimate_font_size(config: FontEngineRunConfig):
  42    style = config.style
  43
  44    # Estimate the font size based on height or width.
  45    if config.glyph_sequence == FontEngineRunConfigGlyphSequence.HORI_DEFAULT:
  46        font_size = round(config.height * style.font_size_ratio)
  47    elif config.glyph_sequence == FontEngineRunConfigGlyphSequence.VERT_DEFAULT:
  48        font_size = round(config.width * style.font_size_ratio)
  49    else:
  50        raise NotImplementedError()
  51
  52    font_size = int(np.clip(font_size, style.font_size_min, style.font_size_max))
  53    return font_size
  54
  55
  56def load_freetype_font_face(
  57    run_config: FontEngineRunConfig,
  58    lcd_compression_factor: Optional[int] = None,
  59):
  60    font_variant = run_config.font_variant
  61    if font_variant.is_ttc:
  62        assert font_variant.ttc_font_index is not None
  63        font_face = freetype.Face(str(font_variant.font_file), index=font_variant.ttc_font_index)
  64    else:
  65        font_face = freetype.Face(str(font_variant.font_file))
  66
  67    font_size = estimate_font_size(run_config)
  68
  69    # 1. "The nominal width, in 26.6 fractional points" and "The 26.6 fixed float format
  70    #    used to define fractional pixel coordinates. Here, 1 unit = 1/64 pixel", hence
  71    #    we need to multiple 64 here.
  72    # 2. `height` defaults to 0 and "If either the character width or
  73    #    height is zero, it is set equal to the other value",
  74    #    hence no need to set the `height` parameter.
  75    # 3. `vres` for vertical resolution, and defaults to 72.
  76    #    Since "pixel_size = point_size * resolution / 72", setting resolution to 72 equalize
  77    #    point_size and pixel_size. Since "If either the horizontal or vertical resolution is zero,
  78    #    it is set equal to the other value" and `vres` defaults to 72, we don't need to set the
  79    #    `vres` parameter.
  80    # 4. `hres` for horizontal resolution, and defaults to 72. If `lcd_compression_factor` is set,
  81    #    we need to cancel the sub-pixel rendering effect, by scaling up the same factor to `hres`.
  82    hres = 72
  83    if lcd_compression_factor is not None:
  84        hres *= lcd_compression_factor
  85    font_face.set_char_size(width=font_size * 64, hres=hres)
  86
  87    return font_face
  88
  89
  90def build_freetype_font_face_lcd_hc_matrix(lcd_compression_factor: int):
  91    # "hc" stands for "horizontal compression".
  92    return freetype.Matrix(
  93        int((1 / lcd_compression_factor) * 0x10000),
  94        int(0.0 * 0x10000),
  95        int(0.0 * 0x10000),
  96        int(1.0 * 0x10000),
  97    )
  98
  99
 100def trim_char_np_image_vert(np_image: np.ndarray):
 101    # (H, W) or (H, W, 3)
 102    np_vert_max: np.ndarray = np.amax(np_image, axis=1)
 103    if np_vert_max.ndim == 2:
 104        np_vert_max = np.amax(np_vert_max, axis=1)
 105    assert np_vert_max.ndim == 1
 106
 107    np_vert_nonzero = np.nonzero(np_vert_max)[0]
 108    if len(np_vert_nonzero) == 0:
 109        raise RuntimeError('trim_char_np_image_vert: empty np_image.')
 110
 111    up = int(np_vert_nonzero[0])
 112    down = int(np_vert_nonzero[-1])
 113
 114    height = np_image.shape[0]
 115    return np_image[up:down + 1], up, height - 1 - down
 116
 117
 118def trim_char_np_image_hori(np_image: np.ndarray):
 119    # (H, W) or (H, W, 3)
 120    np_hori_max: np.ndarray = np.amax(np_image, axis=0)
 121    if np_hori_max.ndim == 2:
 122        np_hori_max = np.amax(np_hori_max, axis=1)
 123    assert np_hori_max.ndim == 1
 124
 125    np_hori_nonzero = np.nonzero(np_hori_max)[0]
 126    if len(np_hori_nonzero) == 0:
 127        raise RuntimeError('trim_char_np_image_hori: empty np_image.')
 128
 129    left = int(np_hori_nonzero[0])
 130    right = int(np_hori_nonzero[-1])
 131
 132    width = np_image.shape[1]
 133    return np_image[:, left:right + 1], left, width - 1 - right
 134
 135
 136def build_char_glyph(
 137    config: FontEngineRunConfig,
 138    char: str,
 139    glyph: Any,
 140    np_image: np.ndarray,
 141):
 142    assert not char.isspace()
 143
 144    # References:
 145    # https://freetype.org/freetype2/docs/tutorial/step2.html
 146    # https://freetype-py.readthedocs.io/en/latest/glyph_slot.html
 147
 148    # "the distance from the baseline to the top-most glyph scanline"
 149    ascent = glyph.bitmap_top
 150
 151    # "It is always zero for horizontal layouts, and positive for vertical layouts."
 152    assert glyph.advance.y == 0
 153
 154    # Trim vertically to get pad_up & pad_down.
 155    np_image, pad_up, pad_down = trim_char_np_image_vert(np_image)
 156    ascent -= pad_up
 157
 158    # "bitmap’s left bearing"
 159    # NOTE: `bitmap_left` could be negative, we simply ignore the negative value.
 160    pad_left = max(0, glyph.bitmap_left)
 161
 162    # "It is always zero for horizontal layouts, and zero for vertical ones."
 163    assert glyph.advance.x > 0
 164    width = np_image.shape[1]
 165    # Aforementioned, 1 unit = 1/64 pixel.
 166    pad_right = round(glyph.advance.x / 64) - pad_left - width
 167    # Apply defensive clipping.
 168    pad_right = max(0, pad_right)
 169
 170    # Trim horizontally to increase pad_left & pad_right.
 171    np_image, pad_left_inc, pad_right_inc = trim_char_np_image_hori(np_image)
 172    pad_left += pad_left_inc
 173    pad_right += pad_right_inc
 174
 175    score_map = None
 176    if np_image.ndim == 2:
 177        # Apply gamma correction.
 178        np_alpha = np.power(
 179            np_image.astype(np.float32) / 255.0,
 180            config.style.glyph_color_gamma,
 181        )
 182        score_map = ScoreMap(mat=np_alpha)
 183
 184    # Reference statistics.
 185    font_variant = config.font_variant
 186    tag_to_font_glyph_info = font_variant.font_glyph_info_collection.tag_to_font_glyph_info
 187
 188    assert char in font_variant.char_to_tags
 189    tags = font_variant.char_to_tags[char]
 190
 191    font_glyph_info = None
 192    for tag in tags:
 193        assert tag in tag_to_font_glyph_info
 194        cur_font_glyph_info = tag_to_font_glyph_info[tag]
 195        if font_glyph_info is None:
 196            font_glyph_info = cur_font_glyph_info
 197        else:
 198            assert font_glyph_info == cur_font_glyph_info
 199
 200    assert font_glyph_info is not None
 201
 202    font_size = estimate_font_size(config)
 203    ref_ascent_plus_pad_up = round(
 204        font_glyph_info.ascent_plus_pad_up_min_to_font_size_ratio * font_size
 205    )
 206    ref_char_height = round(font_glyph_info.height_min_to_font_size_ratio * font_size)
 207    ref_char_width = round(font_glyph_info.width_min_to_font_size_ratio * font_size)
 208
 209    return CharGlyph(
 210        char=char,
 211        image=Image(mat=np_image),
 212        score_map=score_map,
 213        ascent=ascent,
 214        pad_up=pad_up,
 215        pad_down=pad_down,
 216        pad_left=pad_left,
 217        pad_right=pad_right,
 218        ref_ascent_plus_pad_up=ref_ascent_plus_pad_up,
 219        ref_char_height=ref_char_height,
 220        ref_char_width=ref_char_width,
 221    )
 222
 223
 224def render_char_glyphs_from_text(
 225    run_config: FontEngineRunConfig,
 226    font_face: freetype.Face,
 227    func_render_char_glyph: Callable[[FontEngineRunConfig, freetype.Face, str], CharGlyph],
 228    chars: Sequence[str],
 229):
 230    char_glyphs: List[CharGlyph] = []
 231    prev_num_spaces_for_char_glyphs: List[int] = []
 232    num_spaces = 0
 233    for idx, char in enumerate(chars):
 234        if char.isspace():
 235            num_spaces += 1
 236            continue
 237
 238        char_glyphs.append(func_render_char_glyph(run_config, font_face, char))
 239
 240        if idx == 0 and num_spaces > 0:
 241            raise RuntimeError('Leading space(s) detected.')
 242        prev_num_spaces_for_char_glyphs.append(num_spaces)
 243        num_spaces = 0
 244
 245    if num_spaces > 0:
 246        raise RuntimeError('Trailing space(s) detected.')
 247
 248    return char_glyphs, prev_num_spaces_for_char_glyphs
 249
 250
 251def get_kerning_limits_hori_default(
 252    char_glyphs: Sequence[CharGlyph],
 253    prev_num_spaces_for_char_glyphs: Sequence[int],
 254):
 255    assert char_glyphs
 256    ascent_max = max(char_glyph.ascent for char_glyph in char_glyphs)
 257
 258    kerning_limits: List[int] = []
 259
 260    prev_glyph_mask = None
 261    prev_np_glyph_mask = None
 262    prev_glyph_mask_up = None
 263    prev_glyph_mask_down = None
 264    for char_glyph, prev_num_spaces in zip(char_glyphs, prev_num_spaces_for_char_glyphs):
 265        glyph_mask = char_glyph.get_glyph_mask()
 266        np_glyph_mask = glyph_mask.mat
 267        glyph_mask_up = ascent_max - char_glyph.ascent
 268        glyph_mask_down = glyph_mask_up + np_glyph_mask.shape[0] - 1
 269
 270        if prev_num_spaces == 0 and prev_np_glyph_mask is not None:
 271            assert prev_glyph_mask is not None
 272            assert prev_glyph_mask_up is not None
 273            assert prev_glyph_mask_down is not None
 274
 275            overlap_up = max(prev_glyph_mask_up, glyph_mask_up)
 276            overlap_down = min(prev_glyph_mask_down, glyph_mask_down)
 277            if overlap_up <= overlap_down:
 278                overlap_prev_np_glyph_mask = \
 279                    prev_np_glyph_mask[overlap_up - prev_glyph_mask_up:
 280                                       overlap_down - prev_glyph_mask_up + 1]
 281                overlap_np_glyph_mask = \
 282                    np_glyph_mask[overlap_up - glyph_mask_up:
 283                                  overlap_down - glyph_mask_up + 1]
 284
 285                kerning_limit = 1
 286                while kerning_limit < prev_glyph_mask.width / 2 \
 287                        and kerning_limit < glyph_mask.width / 2:
 288                    prev_np_glyph_mask_tail = overlap_prev_np_glyph_mask[:, -kerning_limit:]
 289                    np_glyph_mask_head = overlap_np_glyph_mask[:, :kerning_limit]
 290                    if (prev_np_glyph_mask_tail & np_glyph_mask_head).any():
 291                        # Intersection detected.
 292                        kerning_limit -= 1
 293                        break
 294                    kerning_limit += 1
 295
 296                kerning_limits.append(kerning_limit)
 297
 298            else:
 299                # Not overlapped.
 300                kerning_limits.append(0)
 301
 302        else:
 303            # Isolated by word space or being the first glyph, skip.
 304            kerning_limits.append(0)
 305
 306        prev_glyph_mask = glyph_mask
 307        prev_np_glyph_mask = np_glyph_mask
 308        prev_glyph_mask_up = glyph_mask_up
 309        prev_glyph_mask_down = glyph_mask_down
 310
 311    return kerning_limits
 312
 313
 314def render_char_glyphs_in_text_line(
 315    style: FontEngineRunConfigStyle,
 316    text_line_height: int,
 317    text_line_width: int,
 318    char_glyphs: Sequence[CharGlyph],
 319    char_boxes: Sequence[CharBox],
 320):
 321    # Genrate text line image.
 322    np_image = np.full((text_line_height, text_line_width, 3), 255, dtype=np.uint8)
 323    np_mask = np.zeros((text_line_height, text_line_width), dtype=np.uint8)
 324    score_map = None
 325
 326    if char_glyphs[0].image.mat.ndim == 2:
 327        score_map = ScoreMap.from_shape((text_line_height, text_line_width))
 328
 329        # Default or monochrome.
 330        for char_glyph, char_box in zip(char_glyphs, char_boxes):
 331            assert char_glyph.score_map
 332
 333            char_glyph_mask = char_glyph.get_glyph_mask(box=char_box.box)
 334
 335            # Fill color based on RGBA & alpha.
 336            np_char_image = np.full(
 337                (char_glyph.height, char_glyph.width, 4),
 338                (*style.glyph_color, 0),
 339                dtype=np.uint8,
 340            )
 341            np_char_image[:, :, 3] = (char_glyph.score_map.mat * 255).astype(np.uint8)
 342
 343            # To RGB.
 344            np_char_image = cv.cvtColor(np_char_image, cv.COLOR_RGBA2RGB)
 345
 346            # Paste to text line.
 347            char_glyph_mask.fill_np_array(np_image, np_char_image)
 348            char_glyph_mask.fill_np_array(np_mask, 1)
 349            char_box.box.fill_score_map(
 350                score_map,
 351                char_glyph.score_map,
 352                keep_max_value=True,
 353            )
 354
 355    elif char_glyphs[0].image.mat.ndim == 3:
 356        # LCD.
 357        for char_glyph, char_box in zip(char_glyphs, char_boxes):
 358            char_glyph_mask = char_glyph.get_glyph_mask(box=char_box.box)
 359
 360            # NOTE: the `glyph_color` option is ignored in LCD mode.
 361            # Gamma correction.
 362            np_char_image = np.power(
 363                char_glyph.image.mat / 255.0,
 364                style.glyph_color_gamma,
 365            )
 366            np_char_image = ((1 - np_char_image) * 255).astype(np.uint8)  # type: ignore
 367
 368            # Paste to text line.
 369            char_glyph_mask.fill_np_array(np_image, np_char_image)
 370            char_glyph_mask.fill_np_array(np_mask, 1)
 371
 372    else:
 373        raise NotImplementedError()
 374
 375    return (
 376        Image(mat=np_image),
 377        Mask(mat=np_mask),
 378        score_map,
 379        char_boxes,
 380    )
 381
 382
 383def place_char_glyphs_in_text_line_hori_default(
 384    run_config: FontEngineRunConfig,
 385    char_glyphs: Sequence[CharGlyph],
 386    prev_num_spaces_for_char_glyphs: Sequence[int],
 387    kerning_limits: Sequence[int],
 388    rng: RandomGenerator,
 389):
 390    style = run_config.style
 391
 392    assert char_glyphs
 393    char_widths_avg = np.mean([char_glyph.width for char_glyph in char_glyphs])
 394
 395    char_space_min = char_widths_avg * style.char_space_min
 396    char_space_max = char_widths_avg * style.char_space_max
 397    char_space_mean = char_widths_avg * style.char_space_mean
 398    char_space_std = char_widths_avg * style.char_space_std
 399
 400    word_space_min = char_widths_avg * style.word_space_min
 401    word_space_max = char_widths_avg * style.word_space_max
 402    word_space_mean = char_widths_avg * style.word_space_mean
 403    word_space_std = char_widths_avg * style.word_space_std
 404
 405    ascent_plus_pad_up_max = max(
 406        itertools.chain.from_iterable(
 407            (char_glyph.ascent + char_glyph.pad_up, char_glyph.ref_ascent_plus_pad_up)
 408            for char_glyph in char_glyphs
 409        )
 410    )
 411
 412    text_line_height = max(char_glyph.ref_char_height for char_glyph in char_glyphs)
 413
 414    char_boxes: List[CharBox] = []
 415    hori_offset = 0
 416    for char_idx, (char_glyph, prev_num_spaces, kerning_limit) in enumerate(
 417        zip(
 418            char_glyphs,
 419            prev_num_spaces_for_char_glyphs,
 420            kerning_limits,
 421        )
 422    ):
 423        # "Stick" chars together.
 424        hori_offset -= kerning_limit
 425
 426        # Shift by space.
 427        if prev_num_spaces > 0:
 428            # Random word space(s).
 429            space = 0
 430            for _ in range(prev_num_spaces):
 431                space += round(
 432                    np.clip(
 433                        rng.normal(loc=word_space_mean, scale=word_space_std),
 434                        word_space_min,
 435                        word_space_max,
 436                    )  # type: ignore
 437                )
 438
 439        else:
 440            # Random char space.
 441            space = round(
 442                np.clip(
 443                    rng.normal(loc=char_space_mean, scale=char_space_std),
 444                    char_space_min,
 445                    char_space_max,
 446                )  # type: ignore
 447            )
 448
 449        hori_offset += space
 450
 451        # Place char box.
 452        up = ascent_plus_pad_up_max - char_glyph.ascent
 453        down = up + char_glyph.height - 1
 454
 455        left = hori_offset + char_glyph.pad_left
 456        if char_idx == 0:
 457            # Ignore the leading padding.
 458            left = 0
 459        right = left + char_glyph.width - 1
 460
 461        assert not char_glyph.char.isspace()
 462        char_boxes.append(
 463            CharBox(
 464                char=char_glyph.char,
 465                box=Box(
 466                    up=up,
 467                    down=down,
 468                    left=left,
 469                    right=right,
 470                ),
 471            )
 472        )
 473
 474        # Update the height of text line.
 475        text_line_height = max(text_line_height, down + 1 + char_glyph.pad_down)
 476
 477        # Move offset.
 478        hori_offset = right + 1
 479        if char_idx < len(char_glyphs) - 1:
 480            hori_offset += char_glyph.pad_right
 481
 482    text_line_width = hori_offset
 483
 484    return render_char_glyphs_in_text_line(
 485        style=style,
 486        text_line_height=text_line_height,
 487        text_line_width=text_line_width,
 488        char_glyphs=char_glyphs,
 489        char_boxes=char_boxes,
 490    )
 491
 492
 493def place_char_glyphs_in_text_line_vert_default(
 494    run_config: FontEngineRunConfig,
 495    char_glyphs: Sequence[CharGlyph],
 496    prev_num_spaces_for_char_glyphs: Sequence[int],
 497    rng: RandomGenerator,
 498):
 499    style = run_config.style
 500
 501    assert char_glyphs
 502    char_widths_avg = np.mean([char_glyph.width for char_glyph in char_glyphs])
 503
 504    char_space_min = char_widths_avg * style.char_space_min
 505    char_space_max = char_widths_avg * style.char_space_max
 506    char_space_mean = char_widths_avg * style.char_space_mean
 507    char_space_std = char_widths_avg * style.char_space_std
 508
 509    word_space_min = char_widths_avg * style.word_space_min
 510    word_space_max = char_widths_avg * style.word_space_max
 511    word_space_mean = char_widths_avg * style.word_space_mean
 512    word_space_std = char_widths_avg * style.word_space_std
 513
 514    text_line_width = max(
 515        itertools.chain.from_iterable((
 516            char_glyph.pad_left + char_glyph.width + char_glyph.pad_right,
 517            char_glyph.ref_char_width,
 518        ) for char_glyph in char_glyphs)
 519    )
 520
 521    text_line_width_mid = text_line_width // 2
 522
 523    char_boxes: List[CharBox] = []
 524    vert_offset = 0
 525    for char_idx, (char_glyph, prev_num_spaces) in enumerate(
 526        zip(char_glyphs, prev_num_spaces_for_char_glyphs)
 527    ):
 528        # Shift by space.
 529        if prev_num_spaces > 0:
 530            # Random word space(s).
 531            space = 0
 532            for _ in range(prev_num_spaces):
 533                space += round(
 534                    np.clip(
 535                        rng.normal(loc=word_space_mean, scale=word_space_std),
 536                        word_space_min,
 537                        word_space_max,
 538                    )  # type: ignore
 539                )
 540
 541        else:
 542            # Random char space.
 543            space = round(
 544                np.clip(
 545                    rng.normal(loc=char_space_mean, scale=char_space_std),
 546                    char_space_min,
 547                    char_space_max,
 548                )  # type: ignore
 549            )
 550
 551        vert_offset += space
 552
 553        # Place char box.
 554        up = vert_offset + char_glyph.pad_up
 555        if char_idx == 0:
 556            # Ignore the leading padding.
 557            up = 0
 558
 559        down = up + char_glyph.height - 1
 560
 561        # Vertical align in middle.
 562        left = text_line_width_mid - char_glyph.width // 2
 563        right = left + char_glyph.width - 1
 564
 565        assert not char_glyph.char.isspace()
 566        char_boxes.append(
 567            CharBox(
 568                char=char_glyph.char,
 569                box=Box(
 570                    up=up,
 571                    down=down,
 572                    left=left,
 573                    right=right,
 574                ),
 575            )
 576        )
 577
 578        # Move offset.
 579        vert_offset = down + 1
 580        if char_idx < len(char_glyphs) - 1:
 581            vert_offset += char_glyph.pad_down
 582
 583    text_line_height = vert_offset
 584
 585    return render_char_glyphs_in_text_line(
 586        style=style,
 587        text_line_height=text_line_height,
 588        text_line_width=text_line_width,
 589        char_glyphs=char_glyphs,
 590        char_boxes=char_boxes,
 591    )
 592
 593
 594def resize_and_trim_text_line_hori_default(
 595    run_config: FontEngineRunConfig,
 596    cv_resize_interpolation_enlarge: int,
 597    cv_resize_interpolation_shrink: int,
 598    image: Image,
 599    mask: Mask,
 600    score_map: Optional[ScoreMap],
 601    char_boxes: Sequence[CharBox],
 602    char_glyphs: Sequence[CharGlyph],
 603):
 604    # Resize if image height too small or too large.
 605    is_too_small = (image.height / run_config.height < 0.8)
 606    is_too_large = (image.height > run_config.height)
 607
 608    cv_resize_interpolation = cv_resize_interpolation_enlarge
 609    if is_too_large:
 610        cv_resize_interpolation = cv_resize_interpolation_shrink
 611
 612    if is_too_small or is_too_large:
 613        resized_image = image.to_resized_image(
 614            resized_height=run_config.height,
 615            cv_resize_interpolation=cv_resize_interpolation,
 616        )
 617        resized_mask = mask.to_resized_mask(
 618            resized_height=run_config.height,
 619            cv_resize_interpolation=cv_resize_interpolation,
 620        )
 621        resized_char_boxes = [
 622            char_box.to_conducted_resized_char_box(
 623                shapable_or_shape=image,
 624                resized_height=run_config.height,
 625            ) for char_box in char_boxes
 626        ]
 627
 628        image = resized_image
 629        mask = resized_mask
 630        char_boxes = resized_char_boxes
 631
 632        if score_map:
 633            score_map = score_map.to_resized_score_map(
 634                resized_height=run_config.height,
 635                cv_resize_interpolation=cv_resize_interpolation,
 636            )
 637
 638    # Pad vertically.
 639    if image.height != run_config.height:
 640        pad_vert = run_config.height - image.height
 641        assert pad_vert > 0
 642        pad_up = pad_vert // 2
 643        pad_down = pad_vert - pad_up
 644
 645        np_image = np.full((run_config.height, image.width, 3), 255, dtype=np.uint8)
 646        np_image[pad_up:-pad_down] = image.mat
 647        image.assign_mat(np_image)
 648
 649        np_mask = np.zeros((run_config.height, image.width), dtype=np.uint8)
 650        np_mask[pad_up:-pad_down] = mask.mat
 651        mask.assign_mat(np_mask)
 652
 653        padded_char_boxes = []
 654        for char_box in char_boxes:
 655            box = attrs.evolve(
 656                char_box.box,
 657                up=char_box.up + pad_up,
 658                down=char_box.down + pad_up,
 659            )
 660            padded_char_boxes.append(attrs.evolve(char_box, box=box))
 661        char_boxes = padded_char_boxes
 662
 663        if score_map:
 664            padded_score_map = ScoreMap.from_shape((run_config.height, image.width))
 665            with padded_score_map.writable_context:
 666                padded_score_map.mat[pad_up:-pad_down] = score_map.mat
 667            score_map = padded_score_map
 668
 669    # Trim.
 670    if image.width > run_config.width:
 671        last_char_box_idx = len(char_boxes) - 1
 672        while last_char_box_idx >= 0 and char_boxes[last_char_box_idx].right >= run_config.width:
 673            last_char_box_idx -= 1
 674
 675        if last_char_box_idx == len(char_boxes) - 1:
 676            # Corner case: char_boxes[-1].right < config.width but mage.width > config.width.
 677            # This is caused by glyph padding. The solution is to drop this char.
 678            last_char_box_idx -= 1
 679
 680        if last_char_box_idx < 0 or char_boxes[last_char_box_idx].right >= run_config.width:
 681            # Cannot trim.
 682            return None, None, None, None, -1
 683
 684        last_char_box = char_boxes[last_char_box_idx]
 685        last_char_box_right = last_char_box.right
 686
 687        # Clean up residual pixels.
 688        first_trimed_char_box = char_boxes[last_char_box_idx + 1]
 689        if first_trimed_char_box.left <= last_char_box_right:
 690            first_trimed_char_glyph = char_glyphs[last_char_box_idx + 1]
 691
 692            first_trimed_char_glyph_mask = first_trimed_char_glyph.get_glyph_mask(
 693                box=first_trimed_char_box.box,
 694                enable_resize=True,
 695                cv_resize_interpolation=cv_resize_interpolation,
 696            )
 697            first_trimed_char_glyph_mask.fill_image(image, (255, 255, 255))
 698            first_trimed_char_glyph_mask.fill_mask(mask, 0)
 699
 700            if first_trimed_char_glyph.score_map:
 701                assert score_map
 702
 703                first_trimed_char_score_map = first_trimed_char_glyph.score_map
 704                if first_trimed_char_score_map.shape != first_trimed_char_box.shape:
 705                    first_trimed_char_score_map = first_trimed_char_score_map.to_resized_score_map(
 706                        resized_height=first_trimed_char_box.height,
 707                        resized_width=first_trimed_char_box.width,
 708                        cv_resize_interpolation=cv_resize_interpolation,
 709                    )
 710
 711                last_char_score_map = char_glyphs[last_char_box_idx].score_map
 712                assert last_char_score_map
 713                if last_char_score_map.shape != last_char_box.shape:
 714                    last_char_score_map = last_char_score_map.to_resized_score_map(
 715                        resized_height=last_char_box.height,
 716                        resized_width=last_char_box.width,
 717                        cv_resize_interpolation=cv_resize_interpolation,
 718                    )
 719
 720                first_trimed_char_box.box.fill_score_map(score_map, 0)
 721                last_char_box.box.fill_score_map(
 722                    score_map,
 723                    last_char_score_map,
 724                    keep_max_value=True,
 725                )
 726
 727        char_boxes = char_boxes[:last_char_box_idx + 1]
 728        image.assign_mat(image.mat[:, :last_char_box_right + 1])
 729        mask.assign_mat(mask.mat[:, :last_char_box_right + 1])
 730
 731        if score_map:
 732            score_map.assign_mat(score_map.mat[:, :last_char_box_right + 1])
 733
 734    return image, mask, score_map, char_boxes, cv_resize_interpolation
 735
 736
 737def resize_and_trim_text_line_vert_default(
 738    run_config: FontEngineRunConfig,
 739    cv_resize_interpolation_enlarge: int,
 740    cv_resize_interpolation_shrink: int,
 741    image: Image,
 742    mask: Mask,
 743    score_map: Optional[ScoreMap],
 744    char_boxes: Sequence[CharBox],
 745):
 746    # Resize if image width too small or too large.
 747    is_too_small = (image.width / run_config.width < 0.8)
 748    is_too_large = (image.width > run_config.width)
 749
 750    cv_resize_interpolation = cv_resize_interpolation_enlarge
 751    if is_too_large:
 752        cv_resize_interpolation = cv_resize_interpolation_shrink
 753
 754    if is_too_small or is_too_large:
 755        resized_image = image.to_resized_image(
 756            resized_width=run_config.width,
 757            cv_resize_interpolation=cv_resize_interpolation,
 758        )
 759        resized_mask = mask.to_resized_mask(
 760            resized_width=run_config.width,
 761            cv_resize_interpolation=cv_resize_interpolation,
 762        )
 763        resized_char_boxes = [
 764            char_box.to_conducted_resized_char_box(
 765                shapable_or_shape=image,
 766                resized_width=run_config.width,
 767            ) for char_box in char_boxes
 768        ]
 769
 770        image = resized_image
 771        mask = resized_mask
 772        char_boxes = resized_char_boxes
 773
 774        if score_map:
 775            score_map = score_map.to_resized_score_map(
 776                resized_width=run_config.width,
 777                cv_resize_interpolation=cv_resize_interpolation,
 778            )
 779
 780    # Pad horizontally.
 781    if image.width != run_config.width:
 782        pad_hori = run_config.width - image.width
 783        assert pad_hori > 0
 784        pad_left = pad_hori // 2
 785        pad_right = pad_hori - pad_left
 786
 787        np_image = np.full((image.height, run_config.width, 3), 255, dtype=np.uint8)
 788        np_image[:, pad_left:-pad_right] = image.mat
 789        image.assign_mat(np_image)
 790
 791        np_mask = np.zeros((image.height, run_config.width), dtype=np.uint8)
 792        np_mask[:, pad_left:-pad_right] = mask.mat
 793        mask.assign_mat(np_mask)
 794
 795        padded_char_boxes = []
 796        for char_box in char_boxes:
 797            box = attrs.evolve(
 798                char_box.box,
 799                left=char_box.left + pad_left,
 800                right=char_box.right + pad_right,
 801            )
 802            padded_char_boxes.append(attrs.evolve(char_box, box=box))
 803        char_boxes = padded_char_boxes
 804
 805        if score_map:
 806            padded_score_map = ScoreMap.from_shape((image.height, run_config.width))
 807            padded_score_map.mat[:, pad_left:-pad_right] = score_map.mat
 808            score_map = padded_score_map
 809
 810    # Trim.
 811    if image.height > run_config.height:
 812        last_char_box_idx = len(char_boxes) - 1
 813        while last_char_box_idx >= 0 and char_boxes[last_char_box_idx].down >= run_config.height:
 814            last_char_box_idx -= 1
 815
 816        if last_char_box_idx == len(char_boxes) - 1:
 817            last_char_box_idx -= 1
 818
 819        if last_char_box_idx < 0 or char_boxes[last_char_box_idx].down >= run_config.height:
 820            # Cannot trim.
 821            return None, None, None, None, -1
 822
 823        last_char_box_down = char_boxes[last_char_box_idx].down
 824        char_boxes = char_boxes[:last_char_box_idx + 1]
 825        image.assign_mat(image.mat[:last_char_box_down + 1])
 826        mask.assign_mat(mask.mat[:last_char_box_down + 1])
 827
 828        if score_map:
 829            score_map.assign_mat(score_map.mat[:last_char_box_down + 1])
 830
 831    return image, mask, score_map, char_boxes, cv_resize_interpolation
 832
 833
 834def render_text_line_meta(
 835    run_config: FontEngineRunConfig,
 836    font_face: freetype.Face,
 837    func_render_char_glyph: Callable[[FontEngineRunConfig, freetype.Face, str], CharGlyph],
 838    rng: RandomGenerator,
 839    cv_resize_interpolation_enlarge: int = cv.INTER_CUBIC,
 840    cv_resize_interpolation_shrink: int = cv.INTER_AREA,
 841):
 842    (
 843        char_glyphs,
 844        prev_num_spaces_for_char_glyphs,
 845    ) = render_char_glyphs_from_text(
 846        run_config=run_config,
 847        font_face=font_face,
 848        func_render_char_glyph=func_render_char_glyph,
 849        chars=run_config.chars,
 850    )
 851    if not char_glyphs:
 852        return None
 853
 854    if run_config.glyph_sequence == FontEngineRunConfigGlyphSequence.HORI_DEFAULT:
 855        kerning_limits = get_kerning_limits_hori_default(
 856            char_glyphs,
 857            prev_num_spaces_for_char_glyphs,
 858        )
 859        (
 860            image,
 861            mask,
 862            score_map,
 863            char_boxes,
 864        ) = place_char_glyphs_in_text_line_hori_default(
 865            run_config=run_config,
 866            char_glyphs=char_glyphs,
 867            prev_num_spaces_for_char_glyphs=prev_num_spaces_for_char_glyphs,
 868            kerning_limits=kerning_limits,
 869            rng=rng,
 870        )
 871        (
 872            image,
 873            mask,
 874            score_map,
 875            char_boxes,
 876            cv_resize_interpolation,
 877        ) = resize_and_trim_text_line_hori_default(
 878            run_config=run_config,
 879            cv_resize_interpolation_enlarge=cv_resize_interpolation_enlarge,
 880            cv_resize_interpolation_shrink=cv_resize_interpolation_shrink,
 881            image=image,
 882            mask=mask,
 883            score_map=score_map,
 884            char_boxes=char_boxes,
 885            char_glyphs=char_glyphs,
 886        )
 887        is_hori = True
 888
 889    elif run_config.glyph_sequence == FontEngineRunConfigGlyphSequence.VERT_DEFAULT:
 890        # NOTE: No kerning limit detection for VERT_DEFAULT mode.
 891        (
 892            image,
 893            mask,
 894            score_map,
 895            char_boxes,
 896        ) = place_char_glyphs_in_text_line_vert_default(
 897            run_config=run_config,
 898            char_glyphs=char_glyphs,
 899            prev_num_spaces_for_char_glyphs=prev_num_spaces_for_char_glyphs,
 900            rng=rng,
 901        )
 902        (
 903            image,
 904            mask,
 905            score_map,
 906            char_boxes,
 907            cv_resize_interpolation,
 908        ) = resize_and_trim_text_line_vert_default(
 909            run_config=run_config,
 910            cv_resize_interpolation_enlarge=cv_resize_interpolation_enlarge,
 911            cv_resize_interpolation_shrink=cv_resize_interpolation_shrink,
 912            image=image,
 913            mask=mask,
 914            score_map=score_map,
 915            char_boxes=char_boxes,
 916        )
 917        is_hori = False
 918
 919    else:
 920        raise NotImplementedError()
 921
 922    if image is None:
 923        return None
 924    else:
 925        assert mask is not None
 926        assert char_boxes is not None
 927
 928        char_idx = 0
 929        non_space_count = 0
 930        while char_idx < len(run_config.chars) and non_space_count < len(char_boxes):
 931            if not run_config.chars[char_idx].isspace():
 932                non_space_count += 1
 933            char_idx += 1
 934        assert non_space_count == len(char_boxes)
 935
 936        box = Box.from_shapable(image)
 937        image = image.to_box_attached(box)
 938        mask = mask.to_box_attached(box)
 939        if score_map:
 940            score_map = score_map.to_box_attached(box)
 941
 942        return TextLine(
 943            image=image,
 944            mask=mask,
 945            score_map=score_map,
 946            char_boxes=char_boxes,
 947            char_glyphs=char_glyphs[:len(char_boxes)],
 948            cv_resize_interpolation=cv_resize_interpolation,
 949            font_size=estimate_font_size(run_config),
 950            style=run_config.style,
 951            text=''.join(run_config.chars[:char_idx]),
 952            is_hori=is_hori,
 953            font_variant=run_config.font_variant if run_config.return_font_variant else None,
 954        )
 955
 956
 957class FontFreetypeDefaultEngine(
 958    Engine[
 959        NoneTypeEngineInitConfig,
 960        NoneTypeEngineInitResource,
 961        FontEngineRunConfig,
 962        Optional[TextLine],
 963    ]
 964):  # yapf: disable
 965
 966    @classmethod
 967    def get_type_name(cls) -> str:
 968        return 'freetype_default'
 969
 970    @classmethod
 971    def render_char_glyph(
 972        cls,
 973        run_config: FontEngineRunConfig,
 974        font_face: freetype.Face,
 975        char: str,
 976    ):
 977        load_char_flags = freetype.FT_LOAD_RENDER  # type: ignore
 978        if run_config.style.freetype_force_autohint:
 979            load_char_flags |= freetype.FT_LOAD_FORCE_AUTOHINT  # type: ignore
 980        font_face.load_char(char, load_char_flags)
 981
 982        glyph = font_face.glyph
 983        bitmap = glyph.bitmap
 984
 985        height = bitmap.rows
 986        width = bitmap.width
 987        assert width == bitmap.pitch
 988
 989        # (H, W), [0, 255]
 990        np_image = np.asarray(bitmap.buffer, dtype=np.uint8).reshape(height, width)
 991
 992        return build_char_glyph(run_config, char, glyph, np_image)
 993
 994    def run(self, run_config: FontEngineRunConfig, rng: RandomGenerator) -> Optional[TextLine]:
 995        font_face = load_freetype_font_face(run_config)
 996        return render_text_line_meta(
 997            run_config=run_config,
 998            font_face=font_face,
 999            func_render_char_glyph=self.render_char_glyph,
1000            rng=rng,
1001            cv_resize_interpolation_enlarge=sample_cv_resize_interpolation(rng),
1002            cv_resize_interpolation_shrink=sample_cv_resize_interpolation(
1003                rng,
1004                include_cv_inter_area=True,
1005            ),
1006        )
1007
1008
1009font_freetype_default_engine_executor_factory = EngineExecutorFactory(FontFreetypeDefaultEngine)
1010
1011
1012class FontFreetypeLcdEngine(
1013    Engine[
1014        NoneTypeEngineInitConfig,
1015        NoneTypeEngineInitResource,
1016        FontEngineRunConfig,
1017        Optional[TextLine],
1018    ]
1019):  # yapf: disable
1020
1021    @classmethod
1022    def get_type_name(cls) -> str:
1023        return 'freetype_lcd'
1024
1025    @classmethod
1026    def render_char_glyph(
1027        cls,
1028        run_config: FontEngineRunConfig,
1029        font_face: freetype.Face,
1030        lcd_hc_matrix: freetype.Matrix,
1031        char: str,
1032    ):
1033        load_char_flags = freetype.FT_LOAD_RENDER | freetype.FT_LOAD_TARGET_LCD  # type: ignore
1034        if run_config.style.freetype_force_autohint:
1035            load_char_flags |= freetype.FT_LOAD_FORCE_AUTOHINT  # type: ignore
1036        font_face.set_transform(lcd_hc_matrix, freetype.Vector(0, 0))
1037        font_face.load_char(char, load_char_flags)
1038
1039        glyph = font_face.glyph
1040        bitmap = glyph.bitmap
1041
1042        height = bitmap.rows
1043        pitch = bitmap.pitch
1044        flatten_width = bitmap.width
1045        width = flatten_width // 3
1046
1047        # (H, W, 3), [0, 255]
1048        np_image = np.asarray(bitmap.buffer, dtype=np.uint8).reshape(height, pitch)
1049        np_image = np_image[:, :width * 3].reshape(height, width, 3)
1050
1051        return build_char_glyph(run_config, char, glyph, np_image)
1052
1053    @classmethod
1054    def bind_render_char_glyph(cls, lcd_hc_matrix: freetype.Matrix):
1055        return lambda config, font_face, char: cls.render_char_glyph(
1056            config,
1057            font_face,
1058            lcd_hc_matrix,
1059            char,
1060        )
1061
1062    def run(self, run_config: FontEngineRunConfig, rng: RandomGenerator) -> Optional[TextLine]:
1063        lcd_compression_factor = 10
1064        font_face = load_freetype_font_face(
1065            run_config,
1066            lcd_compression_factor=lcd_compression_factor,
1067        )
1068        lcd_hc_matrix = build_freetype_font_face_lcd_hc_matrix(lcd_compression_factor)
1069        return render_text_line_meta(
1070            run_config=run_config,
1071            font_face=font_face,
1072            func_render_char_glyph=self.bind_render_char_glyph(lcd_hc_matrix),
1073            rng=rng,
1074            cv_resize_interpolation_enlarge=sample_cv_resize_interpolation(rng),
1075            cv_resize_interpolation_shrink=sample_cv_resize_interpolation(
1076                rng,
1077                include_cv_inter_area=True,
1078            ),
1079        )
1080
1081
1082font_freetype_lcd_engine_executor_factory = EngineExecutorFactory(FontFreetypeLcdEngine)
1083
1084
1085class FontFreetypeMonochromeEngine(
1086    Engine[
1087        NoneTypeEngineInitConfig,
1088        NoneTypeEngineInitResource,
1089        FontEngineRunConfig,
1090        Optional[TextLine],
1091    ]
1092):  # yapf: disable
1093
1094    @classmethod
1095    def get_type_name(cls) -> str:
1096        return 'freetype_monochrome'
1097
1098    @classmethod
1099    def render_char_glyph(
1100        cls,
1101        run_config: FontEngineRunConfig,
1102        font_face: freetype.Face,
1103        char: str,
1104    ):
1105        load_char_flags = freetype.FT_LOAD_RENDER | freetype.FT_LOAD_TARGET_MONO  # type: ignore
1106        if run_config.style.freetype_force_autohint:
1107            load_char_flags |= freetype.FT_LOAD_FORCE_AUTOHINT  # type: ignore
1108        font_face.load_char(char, load_char_flags)
1109
1110        glyph = font_face.glyph
1111        bitmap = glyph.bitmap
1112
1113        height = bitmap.rows
1114        width = bitmap.width
1115        pitch = bitmap.pitch
1116
1117        # Performance optimization.
1118        bitmap_buffer = bitmap.buffer
1119
1120        data = []
1121        for row_idx in range(height):
1122            row = []
1123            for pitch_idx in range(pitch):
1124                byte = bitmap_buffer[row_idx * pitch + pitch_idx]
1125                bits = []
1126                for _ in range(8):
1127                    bits.append(int((byte & 1) == 1))
1128                    byte = byte >> 1
1129                row.extend(bit * 255 for bit in reversed(bits))
1130            data.append(row[:width])
1131
1132        np_image = np.asarray(data, dtype=np.uint8)
1133        assert np_image.shape == (height, width)
1134
1135        return build_char_glyph(run_config, char, glyph, np_image)
1136
1137    def run(self, run_config: FontEngineRunConfig, rng: RandomGenerator) -> Optional[TextLine]:
1138        font_face = load_freetype_font_face(run_config)
1139        return render_text_line_meta(
1140            run_config=run_config,
1141            font_face=font_face,
1142            func_render_char_glyph=self.render_char_glyph,
1143            rng=rng,
1144            cv_resize_interpolation_enlarge=cv.INTER_NEAREST_EXACT,
1145            cv_resize_interpolation_shrink=cv.INTER_NEAREST_EXACT,
1146        )
1147
1148
1149font_freetype_monochrome_engine_executor_factory = EngineExecutorFactory(
1150    FontFreetypeMonochromeEngine
1151)