TextBreakIteratorICU.h   [plain text]

/*
 * Copyright (C) 2017 Apple Inc. All rights reserved.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public License
 * along with this library; see the file COPYING.LIB.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 *
 */

#pragma once

#include <unicode/ubrk.h>
#include <wtf/Optional.h>
#include <wtf/text/icu/UTextProviderLatin1.h>

#define USE_ICU_CARET_ITERATOR (PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101200)

namespace WTF {

#if USE_ICU_CARET_ITERATOR
static String caretRules()
{
    static StaticStringImpl caretRuleString(
        // This rule set is based on character-break iterator rules of ICU 57
        // <http://source.icu-project.org/repos/icu/icu/tags/release-57-1/source/data/brkitr/>.
        // The major differences from the original ones are listed below:
        // * Replaced '[\p{Grapheme_Cluster_Break = SpacingMark}]' with '[\p{General_Category = Spacing Mark} - $Extend]' for ICU 3.8 or earlier;
        // * Removed rules that prevent a caret from moving after prepend characters (Bug 24342);
        // * Added rules that prevent a caret from moving after virama signs of Indic languages except Tamil (Bug 15790), and;
        // * Added rules that prevent a caret from moving before Japanese half-width katakara voiced marks.
        // * Added rules for regional indicator symbols.
        "$CR      = [\\p{Grapheme_Cluster_Break = CR}];"
        "$LF      = [\\p{Grapheme_Cluster_Break = LF}];"
        "$Control = [\\p{Grapheme_Cluster_Break = Control}];"
        "$VoiceMarks = [\\uFF9E\\uFF9F];" // Japanese half-width katakana voiced marks
        "$Extend  = [\\p{Grapheme_Cluster_Break = Extend} $VoiceMarks - [\\u0E30 \\u0E32 \\u0E45 \\u0EB0 \\u0EB2]];"
        "$SpacingMark = [[\\p{General_Category = Spacing Mark}] - $Extend];"
        "$L       = [\\p{Grapheme_Cluster_Break = L}];"
        "$V       = [\\p{Grapheme_Cluster_Break = V}];"
        "$T       = [\\p{Grapheme_Cluster_Break = T}];"
        "$LV      = [\\p{Grapheme_Cluster_Break = LV}];"
        "$LVT     = [\\p{Grapheme_Cluster_Break = LVT}];"
        "$Hin0    = [\\u0905-\\u0939];" // Devanagari Letter A,...,Ha
        "$HinV    = \\u094D;" // Devanagari Sign Virama
        "$Hin1    = [\\u0915-\\u0939];" // Devanagari Letter Ka,...,Ha
        "$Ben0    = [\\u0985-\\u09B9];" // Bengali Letter A,...,Ha
        "$BenV    = \\u09CD;" // Bengali Sign Virama
        "$Ben1    = [\\u0995-\\u09B9];" // Bengali Letter Ka,...,Ha
        "$Pan0    = [\\u0A05-\\u0A39];" // Gurmukhi Letter A,...,Ha
        "$PanV    = \\u0A4D;" // Gurmukhi Sign Virama
        "$Pan1    = [\\u0A15-\\u0A39];" // Gurmukhi Letter Ka,...,Ha
        "$Guj0    = [\\u0A85-\\u0AB9];" // Gujarati Letter A,...,Ha
        "$GujV    = \\u0ACD;" // Gujarati Sign Virama
        "$Guj1    = [\\u0A95-\\u0AB9];" // Gujarati Letter Ka,...,Ha
        "$Ori0    = [\\u0B05-\\u0B39];" // Oriya Letter A,...,Ha
        "$OriV    = \\u0B4D;" // Oriya Sign Virama
        "$Ori1    = [\\u0B15-\\u0B39];" // Oriya Letter Ka,...,Ha
        "$Tel0    = [\\u0C05-\\u0C39];" // Telugu Letter A,...,Ha
        "$TelV    = \\u0C4D;" // Telugu Sign Virama
        "$Tel1    = [\\u0C14-\\u0C39];" // Telugu Letter Ka,...,Ha
        "$Kan0    = [\\u0C85-\\u0CB9];" // Kannada Letter A,...,Ha
        "$KanV    = \\u0CCD;" // Kannada Sign Virama
        "$Kan1    = [\\u0C95-\\u0CB9];" // Kannada Letter A,...,Ha
        "$Mal0    = [\\u0D05-\\u0D39];" // Malayalam Letter A,...,Ha
        "$MalV    = \\u0D4D;" // Malayalam Sign Virama
        "$Mal1    = [\\u0D15-\\u0D39];" // Malayalam Letter A,...,Ha
        "$RI      = [\\U0001F1E6-\\U0001F1FF];" // Emoji regional indicators
        "$ZWJ     = \\u200D;" // Zero width joiner
        "$EmojiVar = [\\uFE0F];" // Emoji-style variation selector
        "$EmojiForSeqs = [\\u2640 \\u2642 \\u26F9 \\u2764 \\U0001F308 \\U0001F3C3-\\U0001F3C4 \\U0001F3CA-\\U0001F3CC \\U0001F3F3 \\U0001F441 \\U0001F466-\\U0001F469 \\U0001F46E-\\U0001F46F \\U0001F471 \\U0001F473 \\U0001F477 \\U0001F481-\\U0001F482 \\U0001F486-\\U0001F487 \\U0001F48B \\U0001F575 \\U0001F5E8 \\U0001F645-\\U0001F647 \\U0001F64B \\U0001F64D-\\U0001F64E \\U0001F6A3 \\U0001F6B4-\\U0001F6B6 \\u2695-\\u2696 \\u2708 \\U0001F33E \\U0001F373 \\U0001F393 \\U0001F3A4 \\U0001F3A8 \\U0001F3EB \\U0001F3ED \\U0001F4BB-\\U0001F4BC \\U0001F527 \\U0001F52C \\U0001F680 \\U0001F692 \\U0001F926 \\U0001F937-\\U0001F939 \\U0001F93C-\\U0001F93E];" // Emoji that participate in ZWJ sequences
        "$EmojiForMods = [\\u261D \\u26F9 \\u270A-\\u270D \\U0001F385 \\U0001F3C3-\\U0001F3C4 \\U0001F3CA \\U0001F3CB \\U0001F442-\\U0001F443 \\U0001F446-\\U0001F450 \\U0001F466-\\U0001F478 \\U0001F47C \\U0001F481-\\U0001F483 \\U0001F485-\\U0001F487 \\U0001F4AA \\U0001F575 \\U0001F590 \\U0001F595 \\U0001F596 \\U0001F645-\\U0001F647 \\U0001F64B-\\U0001F64F \\U0001F6A3 \\U0001F6B4-\\U0001F6B6 \\U0001F6C0 \\U0001F918 \\U0001F3C2 \\U0001F3C7 \\U0001F3CC \\U0001F574 \\U0001F57A \\U0001F6CC \\U0001F919-\\U0001F91E \\U0001F926 \\U0001F930 \\U0001F933-\\U0001F939 \\U0001F93C-\\U0001F93E] ;" // Emoji that take Fitzpatrick modifiers
        "$EmojiMods = [\\U0001F3FB-\\U0001F3FF];" // Fitzpatrick modifiers
        "!!chain;"
        "!!RINoChain;"
        "!!forward;"
        "$CR $LF;"
        "$L ($L | $V | $LV | $LVT);"
        "($LV | $V) ($V | $T);"
        "($LVT | $T) $T;"
        "$RI $RI $Extend* / $RI;"
        "$RI $RI $Extend*;"
        "[^$Control $CR $LF] $Extend;"
        "[^$Control $CR $LF] $SpacingMark;"
        "$Hin0 $HinV $Hin1;" // Devanagari Virama (forward)
        "$Ben0 $BenV $Ben1;" // Bengali Virama (forward)
        "$Pan0 $PanV $Pan1;" // Gurmukhi Virama (forward)
        "$Guj0 $GujV $Guj1;" // Gujarati Virama (forward)
        "$Ori0 $OriV $Ori1;" // Oriya Virama (forward)
        "$Tel0 $TelV $Tel1;" // Telugu Virama (forward)
        "$Kan0 $KanV $Kan1;" // Kannada Virama (forward)
        "$Mal0 $MalV $Mal1;" // Malayalam Virama (forward)
        "$ZWJ $EmojiForSeqs;" // Don't break in emoji ZWJ sequences
        "$EmojiForMods $EmojiVar? $EmojiMods;" // Don't break between relevant emoji (possibly with variation selector) and Fitzpatrick modifier
        "!!reverse;"
        "$LF $CR;"
        "($L | $V | $LV | $LVT) $L;"
        "($V | $T) ($LV | $V);"
        "$T ($LVT | $T);"
        "$Extend* $RI $RI / $Extend* $RI $RI;"
        "$Extend* $RI $RI;"
        "$Extend      [^$Control $CR $LF];"
        "$SpacingMark [^$Control $CR $LF];"
        "$Hin1 $HinV $Hin0;" // Devanagari Virama (backward)
        "$Ben1 $BenV $Ben0;" // Bengali Virama (backward)
        "$Pan1 $PanV $Pan0;" // Gurmukhi Virama (backward)
        "$Guj1 $GujV $Guj0;" // Gujarati Virama (backward)
        "$Ori1 $OriV $Ori0;" // Gujarati Virama (backward)
        "$Tel1 $TelV $Tel0;" // Telugu Virama (backward)
        "$Kan1 $KanV $Kan0;" // Kannada Virama (backward)
        "$Mal1 $MalV $Mal0;" // Malayalam Virama (backward)
        "$EmojiForSeqs $ZWJ;" // Don't break in emoji ZWJ sequences
        "$EmojiMods $EmojiVar? $EmojiForMods;" // Don't break between relevant emoji (possibly with variation selector) and Fitzpatrick modifier
        "!!safe_reverse;"
        "$RI $RI+;"
        "[$EmojiVar $EmojiMods]+ $EmojiForMods;"
        "!!safe_forward;"
        "$RI $RI+;"
        "$EmojiForMods [$EmojiVar $EmojiMods]+;"
    );
    return caretRuleString;
}
#endif

class TextBreakIteratorICU {
public:
    enum class Mode {
        Line,
        Character,
#if USE_ICU_CARET_ITERATOR
        Caret,
#endif
    };

    void set8BitText(const LChar* buffer, unsigned length)
    {
        UTextWithBuffer textLocal;
        textLocal.text = UTEXT_INITIALIZER;
        textLocal.text.extraSize = sizeof(textLocal.buffer);
        textLocal.text.pExtra = textLocal.buffer;

        UErrorCode status = U_ZERO_ERROR;
        UText* text = openLatin1UTextProvider(&textLocal, buffer, length, &status);
        ASSERT(U_SUCCESS(status));
        ASSERT(text);

        ubrk_setUText(m_iterator, text, &status);
        ASSERT(U_SUCCESS(status));

        utext_close(text);
    }

    TextBreakIteratorICU(StringView string, Mode mode, const char *locale)
    {
        UBreakIteratorType type;
        switch (mode) {
        case Mode::Line:
            type = UBRK_LINE;
            break;
        case Mode::Character:
            type = UBRK_CHARACTER;
            break;
#if USE_ICU_CARET_ITERATOR
        case Mode::Caret:
            type = UBRK_CHARACTER;
            break;
#endif
        default:
            ASSERT_NOT_REACHED();
            type = UBRK_CHARACTER;
            break;
        }

        bool requiresSet8BitText = string.is8Bit();

        const UChar *text = requiresSet8BitText ? nullptr : string.characters16();
        int32_t textLength = requiresSet8BitText ? 0 : string.length();

        // FIXME: Handle weak / normal / strict line breaking.
        UErrorCode status = U_ZERO_ERROR;
#if USE_ICU_CARET_ITERATOR
        if (mode == Mode::Caret) {
            static NeverDestroyed<String> caretRules = WTF::caretRules();
            static NeverDestroyed<StringView::UpconvertedCharacters> upconvertedRules = StringView(caretRules).upconvertedCharacters();
            UParseError parseError;
            m_iterator = ubrk_openRules(upconvertedRules.get(), caretRules.get().length(), text, textLength, &parseError, &status);
        } else
#endif
            m_iterator = ubrk_open(type, locale, text, textLength, &status);
        ASSERT(U_SUCCESS(status));

        if (requiresSet8BitText)
            set8BitText(string.characters8(), string.length());
    }

    TextBreakIteratorICU() = delete;
    TextBreakIteratorICU(const TextBreakIteratorICU&) = delete;

    TextBreakIteratorICU(TextBreakIteratorICU&& other)
        : m_iterator(other.m_iterator)
    {
        other.m_iterator = nullptr;
    }

    TextBreakIteratorICU& operator=(const TextBreakIteratorICU&) = delete;

    TextBreakIteratorICU& operator=(TextBreakIteratorICU&& other)
    {
        if (m_iterator)
            ubrk_close(m_iterator);
        m_iterator = other.m_iterator;
        other.m_iterator = nullptr;
        return *this;
    }

    ~TextBreakIteratorICU()
    {
        if (m_iterator)
            ubrk_close(m_iterator);
    }

    void setText(StringView string)
    {
        if (string.is8Bit()) {
            set8BitText(string.characters8(), string.length());
            return;
        }
        UErrorCode status = U_ZERO_ERROR;
        ubrk_setText(m_iterator, string.characters16(), string.length(), &status);
        ASSERT(U_SUCCESS(status));
    }

    std::optional<unsigned> preceding(unsigned location) const
    {
        auto result = ubrk_preceding(m_iterator, location);
        if (result == UBRK_DONE)
            return { };
        return result;
    }

    std::optional<unsigned> following(unsigned location) const
    {
        auto result = ubrk_following(m_iterator, location);
        if (result == UBRK_DONE)
            return { };
        return result;
    }

    bool isBoundary(unsigned location) const
    {
        return ubrk_isBoundary(m_iterator, location);
    }

private:
    UBreakIterator* m_iterator;
};

}