uitercollationiterator.cpp   [plain text]

/*
*******************************************************************************
* Copyright (C) 2012-2014, International Business Machines
* Corporation and others.  All Rights Reserved.
*******************************************************************************
* uitercollationiterator.cpp
*
* created on: 2012sep23 (from utf16collationiterator.cpp)
* created by: Markus W. Scherer
*/

#include "unicode/utypes.h"

#if !UCONFIG_NO_COLLATION

#include "unicode/uiter.h"
#include "charstr.h"
#include "cmemory.h"
#include "collation.h"
#include "collationdata.h"
#include "collationfcd.h"
#include "collationiterator.h"
#include "normalizer2impl.h"
#include "uassert.h"
#include "uitercollationiterator.h"

U_NAMESPACE_BEGIN

UIterCollationIterator::~UIterCollationIterator() {}

void
UIterCollationIterator::resetToOffset(int32_t newOffset) {
    reset();
    iter.move(&iter, newOffset, UITER_START);
}

int32_t
UIterCollationIterator::getOffset() const {
    return iter.getIndex(&iter, UITER_CURRENT);
}

uint32_t
UIterCollationIterator::handleNextCE32(UChar32 &c, UErrorCode & /*errorCode*/) {
    c = iter.next(&iter);
    if(c < 0) {
        return Collation::FALLBACK_CE32;
    }
    return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c);
}

UChar
UIterCollationIterator::handleGetTrailSurrogate() {
    UChar32 trail = iter.next(&iter);
    if(!U16_IS_TRAIL(trail) && trail >= 0) { iter.previous(&iter); }
    return (UChar)trail;
}

UChar32
UIterCollationIterator::nextCodePoint(UErrorCode & /*errorCode*/) {
    return uiter_next32(&iter);
}

UChar32
UIterCollationIterator::previousCodePoint(UErrorCode & /*errorCode*/) {
    return uiter_previous32(&iter);
}

void
UIterCollationIterator::forwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) {
    while(num > 0 && (uiter_next32(&iter)) >= 0) {
        --num;
    }
}

void
UIterCollationIterator::backwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) {
    while(num > 0 && (uiter_previous32(&iter)) >= 0) {
        --num;
    }
}

// FCDUIterCollationIterator ----------------------------------------------- ***

FCDUIterCollationIterator::~FCDUIterCollationIterator() {}

void
FCDUIterCollationIterator::resetToOffset(int32_t newOffset) {
    UIterCollationIterator::resetToOffset(newOffset);
    start = newOffset;
    state = ITER_CHECK_FWD;
}

int32_t
FCDUIterCollationIterator::getOffset() const {
    if(state <= ITER_CHECK_BWD) {
        return iter.getIndex(&iter, UITER_CURRENT);
    } else if(state == ITER_IN_FCD_SEGMENT) {
        return pos;
    } else if(pos == 0) {
        return start;
    } else {
        return limit;
    }
}

uint32_t
FCDUIterCollationIterator::handleNextCE32(UChar32 &c, UErrorCode &errorCode) {
    for(;;) {
        if(state == ITER_CHECK_FWD) {
            c = iter.next(&iter);
            if(c < 0) {
                return Collation::FALLBACK_CE32;
            }
            if(CollationFCD::hasTccc(c)) {
                if(CollationFCD::maybeTibetanCompositeVowel(c) ||
                        CollationFCD::hasLccc(iter.current(&iter))) {
                    iter.previous(&iter);
                    if(!nextSegment(errorCode)) {
                        c = U_SENTINEL;
                        return Collation::FALLBACK_CE32;
                    }
                    continue;
                }
            }
            break;
        } else if(state == ITER_IN_FCD_SEGMENT && pos != limit) {
            c = iter.next(&iter);
            ++pos;
            U_ASSERT(c >= 0);
            break;
        } else if(state >= IN_NORM_ITER_AT_LIMIT && pos != normalized.length()) {
            c = normalized[pos++];
            break;
        } else {
            switchToForward();
        }
    }
    return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c);
}

UChar
FCDUIterCollationIterator::handleGetTrailSurrogate() {
    if(state <= ITER_IN_FCD_SEGMENT) {
        UChar32 trail = iter.next(&iter);
        if(U16_IS_TRAIL(trail)) {
            if(state == ITER_IN_FCD_SEGMENT) { ++pos; }
        } else if(trail >= 0) {
            iter.previous(&iter);
        }
        return (UChar)trail;
    } else {
        U_ASSERT(pos < normalized.length());
        UChar trail;
        if(U16_IS_TRAIL(trail = normalized[pos])) { ++pos; }
        return trail;
    }
}

UChar32
FCDUIterCollationIterator::nextCodePoint(UErrorCode &errorCode) {
    UChar32 c;
    for(;;) {
        if(state == ITER_CHECK_FWD) {
            c = iter.next(&iter);
            if(c < 0) {
                return c;
            }
            if(CollationFCD::hasTccc(c)) {
                if(CollationFCD::maybeTibetanCompositeVowel(c) ||
                        CollationFCD::hasLccc(iter.current(&iter))) {
                    iter.previous(&iter);
                    if(!nextSegment(errorCode)) {
                        return U_SENTINEL;
                    }
                    continue;
                }
            }
            if(U16_IS_LEAD(c)) {
                UChar32 trail = iter.next(&iter);
                if(U16_IS_TRAIL(trail)) {
                    return U16_GET_SUPPLEMENTARY(c, trail);
                } else if(trail >= 0) {
                    iter.previous(&iter);
                }
            }
            return c;
        } else if(state == ITER_IN_FCD_SEGMENT && pos != limit) {
            c = uiter_next32(&iter);
            pos += U16_LENGTH(c);
            U_ASSERT(c >= 0);
            return c;
        } else if(state >= IN_NORM_ITER_AT_LIMIT && pos != normalized.length()) {
            c = normalized.char32At(pos);
            pos += U16_LENGTH(c);
            return c;
        } else {
            switchToForward();
        }
    }
}

UChar32
FCDUIterCollationIterator::previousCodePoint(UErrorCode &errorCode) {
    UChar32 c;
    for(;;) {
        if(state == ITER_CHECK_BWD) {
            c = iter.previous(&iter);
            if(c < 0) {
                start = pos = 0;
                state = ITER_IN_FCD_SEGMENT;
                return U_SENTINEL;
            }
            if(CollationFCD::hasLccc(c)) {
                UChar32 prev = U_SENTINEL;
                if(CollationFCD::maybeTibetanCompositeVowel(c) ||
                        CollationFCD::hasTccc(prev = iter.previous(&iter))) {
                    iter.next(&iter);
                    if(prev >= 0) {
                        iter.next(&iter);
                    }
                    if(!previousSegment(errorCode)) {
                        return U_SENTINEL;
                    }
                    continue;
                }
                // hasLccc(trail)=true for all trail surrogates
                if(U16_IS_TRAIL(c)) {
                    if(prev < 0) {
                        prev = iter.previous(&iter);
                    }
                    if(U16_IS_LEAD(prev)) {
                        return U16_GET_SUPPLEMENTARY(prev, c);
                    }
                }
                if(prev >= 0) {
                    iter.next(&iter);
                }
            }
            return c;
        } else if(state == ITER_IN_FCD_SEGMENT && pos != start) {
            c = uiter_previous32(&iter);
            pos -= U16_LENGTH(c);
            U_ASSERT(c >= 0);
            return c;
        } else if(state >= IN_NORM_ITER_AT_LIMIT && pos != 0) {
            c = normalized.char32At(pos - 1);
            pos -= U16_LENGTH(c);
            return c;
        } else {
            switchToBackward();
        }
    }
}

void
FCDUIterCollationIterator::forwardNumCodePoints(int32_t num, UErrorCode &errorCode) {
    // Specify the class to avoid a virtual-function indirection.
    // In Java, we would declare this class final.
    while(num > 0 && FCDUIterCollationIterator::nextCodePoint(errorCode) >= 0) {
        --num;
    }
}

void
FCDUIterCollationIterator::backwardNumCodePoints(int32_t num, UErrorCode &errorCode) {
    // Specify the class to avoid a virtual-function indirection.
    // In Java, we would declare this class final.
    while(num > 0 && FCDUIterCollationIterator::previousCodePoint(errorCode) >= 0) {
        --num;
    }
}

void
FCDUIterCollationIterator::switchToForward() {
    U_ASSERT(state == ITER_CHECK_BWD ||
             (state == ITER_IN_FCD_SEGMENT && pos == limit) ||
             (state >= IN_NORM_ITER_AT_LIMIT && pos == normalized.length()));
    if(state == ITER_CHECK_BWD) {
        // Turn around from backward checking.
        start = pos = iter.getIndex(&iter, UITER_CURRENT);
        if(pos == limit) {
            state = ITER_CHECK_FWD;  // Check forward.
        } else {  // pos < limit
            state = ITER_IN_FCD_SEGMENT;  // Stay in FCD segment.
        }
    } else {
        // Reached the end of the FCD segment.
        if(state == ITER_IN_FCD_SEGMENT) {
            // The input text segment is FCD, extend it forward.
        } else {
            // The input text segment needed to be normalized.
            // Switch to checking forward from it.
            if(state == IN_NORM_ITER_AT_START) {
                iter.move(&iter, limit - start, UITER_CURRENT);
            }
            start = limit;
        }
        state = ITER_CHECK_FWD;
    }
}

UBool
FCDUIterCollationIterator::nextSegment(UErrorCode &errorCode) {
    if(U_FAILURE(errorCode)) { return FALSE; }
    U_ASSERT(state == ITER_CHECK_FWD);
    // The input text [start..(iter index)[ passes the FCD check.
    pos = iter.getIndex(&iter, UITER_CURRENT);
    // Collect the characters being checked, in case they need to be normalized.
    UnicodeString s;
    uint8_t prevCC = 0;
    for(;;) {
        // Fetch the next character and its fcd16 value.
        UChar32 c = uiter_next32(&iter);
        if(c < 0) { break; }
        uint16_t fcd16 = nfcImpl.getFCD16(c);
        uint8_t leadCC = (uint8_t)(fcd16 >> 8);
        if(leadCC == 0 && !s.isEmpty()) {
            // FCD boundary before this character.
            uiter_previous32(&iter);
            break;
        }
        s.append(c);
        if(leadCC != 0 && (prevCC > leadCC || CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) {
            // Fails FCD check. Find the next FCD boundary and normalize.
            for(;;) {
                c = uiter_next32(&iter);
                if(c < 0) { break; }
                if(nfcImpl.getFCD16(c) <= 0xff) {
                    uiter_previous32(&iter);
                    break;
                }
                s.append(c);
            }
            if(!normalize(s, errorCode)) { return FALSE; }
            start = pos;
            limit = pos + s.length();
            state = IN_NORM_ITER_AT_LIMIT;
            pos = 0;
            return TRUE;
        }
        prevCC = (uint8_t)fcd16;
        if(prevCC == 0) {
            // FCD boundary after the last character.
            break;
        }
    }
    limit = pos + s.length();
    U_ASSERT(pos != limit);
    iter.move(&iter, -s.length(), UITER_CURRENT);
    state = ITER_IN_FCD_SEGMENT;
    return TRUE;
}

void
FCDUIterCollationIterator::switchToBackward() {
    U_ASSERT(state == ITER_CHECK_FWD ||
             (state == ITER_IN_FCD_SEGMENT && pos == start) ||
             (state >= IN_NORM_ITER_AT_LIMIT && pos == 0));
    if(state == ITER_CHECK_FWD) {
        // Turn around from forward checking.
        limit = pos = iter.getIndex(&iter, UITER_CURRENT);
        if(pos == start) {
            state = ITER_CHECK_BWD;  // Check backward.
        } else {  // pos > start
            state = ITER_IN_FCD_SEGMENT;  // Stay in FCD segment.
        }
    } else {
        // Reached the start of the FCD segment.
        if(state == ITER_IN_FCD_SEGMENT) {
            // The input text segment is FCD, extend it backward.
        } else {
            // The input text segment needed to be normalized.
            // Switch to checking backward from it.
            if(state == IN_NORM_ITER_AT_LIMIT) {
                iter.move(&iter, start - limit, UITER_CURRENT);
            }
            limit = start;
        }
        state = ITER_CHECK_BWD;
    }
}

UBool
FCDUIterCollationIterator::previousSegment(UErrorCode &errorCode) {
    if(U_FAILURE(errorCode)) { return FALSE; }
    U_ASSERT(state == ITER_CHECK_BWD);
    // The input text [(iter index)..limit[ passes the FCD check.
    pos = iter.getIndex(&iter, UITER_CURRENT);
    // Collect the characters being checked, in case they need to be normalized.
    UnicodeString s;
    uint8_t nextCC = 0;
    for(;;) {
        // Fetch the previous character and its fcd16 value.
        UChar32 c = uiter_previous32(&iter);
        if(c < 0) { break; }
        uint16_t fcd16 = nfcImpl.getFCD16(c);
        uint8_t trailCC = (uint8_t)fcd16;
        if(trailCC == 0 && !s.isEmpty()) {
            // FCD boundary after this character.
            uiter_next32(&iter);
            break;
        }
        s.append(c);
        if(trailCC != 0 && ((nextCC != 0 && trailCC > nextCC) ||
                            CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) {
            // Fails FCD check. Find the previous FCD boundary and normalize.
            while(fcd16 > 0xff) {
                c = uiter_previous32(&iter);
                if(c < 0) { break; }
                fcd16 = nfcImpl.getFCD16(c);
                if(fcd16 == 0) {
                    (void)uiter_next32(&iter);
                    break;
                }
                s.append(c);
            }
            s.reverse();
            if(!normalize(s, errorCode)) { return FALSE; }
            limit = pos;
            start = pos - s.length();
            state = IN_NORM_ITER_AT_START;
            pos = normalized.length();
            return TRUE;
        }
        nextCC = (uint8_t)(fcd16 >> 8);
        if(nextCC == 0) {
            // FCD boundary before the following character.
            break;
        }
    }
    start = pos - s.length();
    U_ASSERT(pos != start);
    iter.move(&iter, s.length(), UITER_CURRENT);
    state = ITER_IN_FCD_SEGMENT;
    return TRUE;
}

UBool
FCDUIterCollationIterator::normalize(const UnicodeString &s, UErrorCode &errorCode) {
    // NFD without argument checking.
    U_ASSERT(U_SUCCESS(errorCode));
    nfcImpl.decompose(s, normalized, errorCode);
    return U_SUCCESS(errorCode);
}

U_NAMESPACE_END

#endif  // !UCONFIG_NO_COLLATION