/* *************************************************************************** * Copyright (C) 1999-2012 International Business Machines Corporation * and others. All rights reserved. *************************************************************************** */ // // file: rbbi.c Contains the implementation of the rule based break iterator // runtime engine and the API implementation for // class RuleBasedBreakIterator // #include <typeinfo> // for 'typeid' to work #include "unicode/utypes.h" #if !UCONFIG_NO_BREAK_ITERATION #include "unicode/rbbi.h" #include "unicode/schriter.h" #include "unicode/uchriter.h" #include "unicode/udata.h" #include "unicode/uclean.h" #include "rbbidata.h" #include "rbbirb.h" #include "cmemory.h" #include "cstring.h" #include "umutex.h" #include "ucln_cmn.h" #include "brkeng.h" #include "uassert.h" #include "uvector.h" // if U_LOCAL_SERVICE_HOOK is defined, then localsvc.cpp is expected to be included. #if U_LOCAL_SERVICE_HOOK #include "localsvc.h" #endif #ifdef RBBI_DEBUG static UBool fTrace = FALSE; #endif U_NAMESPACE_BEGIN // The state number of the starting state #define START_STATE 1 // The state-transition value indicating "stop" #define STOP_STATE 0 UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator) //======================================================================= // constructors //======================================================================= /** * Constructs a RuleBasedBreakIterator that uses the already-created * tables object that is passed in as a parameter. */ RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status) { init(); fData = new RBBIDataWrapper(data, status); // status checked in constructor if (U_FAILURE(status)) {return;} if(fData == 0) { status = U_MEMORY_ALLOCATION_ERROR; return; } } /** * Same as above but does not adopt memory */ RuleBasedBreakIterator::RuleBasedBreakIterator(const RBBIDataHeader* data, enum EDontAdopt, UErrorCode &status) { init(); fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status); // status checked in constructor if (U_FAILURE(status)) {return;} if(fData == 0) { status = U_MEMORY_ALLOCATION_ERROR; return; } } // // Construct from precompiled binary rules (tables). This constructor is public API, // taking the rules as a (const uint8_t *) to match the type produced by getBinaryRules(). // RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules, uint32_t ruleLength, UErrorCode &status) { init(); if (U_FAILURE(status)) { return; } if (compiledRules == NULL || ruleLength < sizeof(RBBIDataHeader)) { status = U_ILLEGAL_ARGUMENT_ERROR; return; } const RBBIDataHeader *data = (const RBBIDataHeader *)compiledRules; if (data->fLength > ruleLength) { status = U_ILLEGAL_ARGUMENT_ERROR; return; } fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status); if (U_FAILURE(status)) {return;} if(fData == 0) { status = U_MEMORY_ALLOCATION_ERROR; return; } } //------------------------------------------------------------------------------- // // Constructor from a UDataMemory handle to precompiled break rules // stored in an ICU data file. // //------------------------------------------------------------------------------- RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status) { init(); fData = new RBBIDataWrapper(udm, status); // status checked in constructor if (U_FAILURE(status)) {return;} if(fData == 0) { status = U_MEMORY_ALLOCATION_ERROR; return; } } //------------------------------------------------------------------------------- // // Constructor from a set of rules supplied as a string. // //------------------------------------------------------------------------------- RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules, UParseError &parseError, UErrorCode &status) { init(); if (U_FAILURE(status)) {return;} RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *) RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status); // Note: This is a bit awkward. The RBBI ruleBuilder has a factory method that // creates and returns a complete RBBI. From here, in a constructor, we // can't just return the object created by the builder factory, hence // the assignment of the factory created object to "this". if (U_SUCCESS(status)) { *this = *bi; delete bi; } } //------------------------------------------------------------------------------- // // Default Constructor. Create an empty shell that can be set up later. // Used when creating a RuleBasedBreakIterator from a set // of rules. //------------------------------------------------------------------------------- RuleBasedBreakIterator::RuleBasedBreakIterator() { init(); } //------------------------------------------------------------------------------- // // Copy constructor. Will produce a break iterator with the same behavior, // and which iterates over the same text, as the one passed in. // //------------------------------------------------------------------------------- RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other) : BreakIterator(other) { this->init(); *this = other; } /** * Destructor */ RuleBasedBreakIterator::~RuleBasedBreakIterator() { if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { // fCharIter was adopted from the outside. delete fCharIter; } fCharIter = NULL; delete fSCharIter; fCharIter = NULL; delete fDCharIter; fDCharIter = NULL; utext_close(fText); if (fData != NULL) { fData->removeReference(); fData = NULL; } if (fCachedBreakPositions) { uprv_free(fCachedBreakPositions); fCachedBreakPositions = NULL; } if (fLanguageBreakEngines) { delete fLanguageBreakEngines; fLanguageBreakEngines = NULL; } if (fUnhandledBreakEngine) { delete fUnhandledBreakEngine; fUnhandledBreakEngine = NULL; } } /** * Assignment operator. Sets this iterator to have the same behavior, * and iterate over the same text, as the one passed in. */ RuleBasedBreakIterator& RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) { if (this == &that) { return *this; } reset(); // Delete break cache information fBreakType = that.fBreakType; if (fLanguageBreakEngines != NULL) { delete fLanguageBreakEngines; fLanguageBreakEngines = NULL; // Just rebuild for now } // TODO: clone fLanguageBreakEngines from "that" UErrorCode status = U_ZERO_ERROR; fText = utext_clone(fText, that.fText, FALSE, TRUE, &status); if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { delete fCharIter; } fCharIter = NULL; if (that.fCharIter != NULL ) { // This is a little bit tricky - it will intially appear that // this->fCharIter is adopted, even if that->fCharIter was // not adopted. That's ok. fCharIter = that.fCharIter->clone(); } if (fData != NULL) { fData->removeReference(); fData = NULL; } if (that.fData != NULL) { fData = that.fData->addReference(); } return *this; } //----------------------------------------------------------------------------- // // init() Shared initialization routine. Used by all the constructors. // Initializes all fields, leaving the object in a consistent state. // //----------------------------------------------------------------------------- void RuleBasedBreakIterator::init() { UErrorCode status = U_ZERO_ERROR; fBufferClone = FALSE; fText = utext_openUChars(NULL, NULL, 0, &status); fCharIter = NULL; fSCharIter = NULL; fDCharIter = NULL; fData = NULL; fLastRuleStatusIndex = 0; fLastStatusIndexValid = TRUE; fDictionaryCharCount = 0; fBreakType = UBRK_WORD; // Defaulting BreakType to word gives reasonable // dictionary behavior for Break Iterators that are // built from rules. Even better would be the ability to // declare the type in the rules. fCachedBreakPositions = NULL; fLanguageBreakEngines = NULL; fUnhandledBreakEngine = NULL; fNumCachedBreakPositions = 0; fPositionInCache = 0; #ifdef RBBI_DEBUG static UBool debugInitDone = FALSE; if (debugInitDone == FALSE) { char *debugEnv = getenv("U_RBBIDEBUG"); if (debugEnv && uprv_strstr(debugEnv, "trace")) { fTrace = TRUE; } debugInitDone = TRUE; } #endif } //----------------------------------------------------------------------------- // // clone - Returns a newly-constructed RuleBasedBreakIterator with the same // behavior, and iterating over the same text, as this one. // Virtual function: does the right thing with subclasses. // //----------------------------------------------------------------------------- BreakIterator* RuleBasedBreakIterator::clone(void) const { return new RuleBasedBreakIterator(*this); } /** * Equality operator. Returns TRUE if both BreakIterators are of the * same class, have the same behavior, and iterate over the same text. */ UBool RuleBasedBreakIterator::operator==(const BreakIterator& that) const { if (typeid(*this) != typeid(that)) { return FALSE; } const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that; if (!utext_equals(fText, that2.fText)) { // The two break iterators are operating on different text, // or have a different interation position. return FALSE; }; // TODO: need a check for when in a dictionary region at different offsets. if (that2.fData == fData || (fData != NULL && that2.fData != NULL && *that2.fData == *fData)) { // The two break iterators are using the same rules. return TRUE; } return FALSE; } /** * Compute a hash code for this BreakIterator * @return A hash code */ int32_t RuleBasedBreakIterator::hashCode(void) const { int32_t hash = 0; if (fData != NULL) { hash = fData->hashCode(); } return hash; } void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) { if (U_FAILURE(status)) { return; } reset(); fText = utext_clone(fText, ut, FALSE, TRUE, &status); // Set up a dummy CharacterIterator to be returned if anyone // calls getText(). With input from UText, there is no reasonable // way to return a characterIterator over the actual input text. // Return one over an empty string instead - this is the closest // we can come to signaling a failure. // (GetText() is obsolete, this failure is sort of OK) if (fDCharIter == NULL) { static const UChar c = 0; fDCharIter = new UCharCharacterIterator(&c, 0); if (fDCharIter == NULL) { status = U_MEMORY_ALLOCATION_ERROR; return; } } if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { // existing fCharIter was adopted from the outside. Delete it now. delete fCharIter; } fCharIter = fDCharIter; this->first(); } UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const { UText *result = utext_clone(fillIn, fText, FALSE, TRUE, &status); return result; } /** * Returns the description used to create this iterator */ const UnicodeString& RuleBasedBreakIterator::getRules() const { if (fData != NULL) { return fData->getRuleSourceString(); } else { static const UnicodeString *s; if (s == NULL) { // TODO: something more elegant here. // perhaps API should return the string by value. // Note: thread unsafe init & leak are semi-ok, better than // what was before. Sould be cleaned up, though. s = new UnicodeString; } return *s; } } //======================================================================= // BreakIterator overrides //======================================================================= /** * Return a CharacterIterator over the text being analyzed. */ CharacterIterator& RuleBasedBreakIterator::getText() const { return *fCharIter; } /** * Set the iterator to analyze a new piece of text. This function resets * the current iteration position to the beginning of the text. * @param newText An iterator over the text to analyze. */ void RuleBasedBreakIterator::adoptText(CharacterIterator* newText) { // If we are holding a CharacterIterator adopted from a // previous call to this function, delete it now. if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { delete fCharIter; } fCharIter = newText; UErrorCode status = U_ZERO_ERROR; reset(); if (newText==NULL || newText->startIndex() != 0) { // startIndex !=0 wants to be an error, but there's no way to report it. // Make the iterator text be an empty string. fText = utext_openUChars(fText, NULL, 0, &status); } else { fText = utext_openCharacterIterator(fText, newText, &status); } this->first(); } /** * Set the iterator to analyze a new piece of text. This function resets * the current iteration position to the beginning of the text. * @param newText An iterator over the text to analyze. */ void RuleBasedBreakIterator::setText(const UnicodeString& newText) { UErrorCode status = U_ZERO_ERROR; reset(); fText = utext_openConstUnicodeString(fText, &newText, &status); // Set up a character iterator on the string. // Needed in case someone calls getText(). // Can not, unfortunately, do this lazily on the (probably never) // call to getText(), because getText is const. if (fSCharIter == NULL) { fSCharIter = new StringCharacterIterator(newText); } else { fSCharIter->setText(newText); } if (fCharIter!=fSCharIter && fCharIter!=fDCharIter) { // old fCharIter was adopted from the outside. Delete it. delete fCharIter; } fCharIter = fSCharIter; this->first(); } /** * Provide a new UText for the input text. Must reference text with contents identical * to the original. * Intended for use with text data originating in Java (garbage collected) environments * where the data may be moved in memory at arbitrary times. */ RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, UErrorCode &status) { if (U_FAILURE(status)) { return *this; } if (input == NULL) { status = U_ILLEGAL_ARGUMENT_ERROR; return *this; } int64_t pos = utext_getNativeIndex(fText); // Shallow read-only clone of the new UText into the existing input UText fText = utext_clone(fText, input, FALSE, TRUE, &status); if (U_FAILURE(status)) { return *this; } utext_setNativeIndex(fText, pos); if (utext_getNativeIndex(fText) != pos) { // Sanity check. The new input utext is supposed to have the exact same // contents as the old. If we can't set to the same position, it doesn't. // The contents underlying the old utext might be invalid at this point, // so it's not safe to check directly. status = U_ILLEGAL_ARGUMENT_ERROR; } return *this; } /** * Sets the current iteration position to the beginning of the text. * @return The offset of the beginning of the text. */ int32_t RuleBasedBreakIterator::first(void) { reset(); fLastRuleStatusIndex = 0; fLastStatusIndexValid = TRUE; //if (fText == NULL) // return BreakIterator::DONE; utext_setNativeIndex(fText, 0); return 0; } /** * Sets the current iteration position to the end of the text. * @return The text's past-the-end offset. */ int32_t RuleBasedBreakIterator::last(void) { reset(); if (fText == NULL) { fLastRuleStatusIndex = 0; fLastStatusIndexValid = TRUE; return BreakIterator::DONE; } fLastStatusIndexValid = FALSE; int32_t pos = (int32_t)utext_nativeLength(fText); utext_setNativeIndex(fText, pos); return pos; } /** * Advances the iterator either forward or backward the specified number of steps. * Negative values move backward, and positive values move forward. This is * equivalent to repeatedly calling next() or previous(). * @param n The number of steps to move. The sign indicates the direction * (negative is backwards, and positive is forwards). * @return The character offset of the boundary position n boundaries away from * the current one. */ int32_t RuleBasedBreakIterator::next(int32_t n) { int32_t result = current(); while (n > 0) { result = next(); --n; } while (n < 0) { result = previous(); ++n; } return result; } /** * Advances the iterator to the next boundary position. * @return The position of the first boundary after this one. */ int32_t RuleBasedBreakIterator::next(void) { // if we have cached break positions and we're still in the range // covered by them, just move one step forward in the cache if (fCachedBreakPositions != NULL) { if (fPositionInCache < fNumCachedBreakPositions - 1) { ++fPositionInCache; int32_t pos = fCachedBreakPositions[fPositionInCache]; utext_setNativeIndex(fText, pos); return pos; } else { reset(); } } int32_t startPos = current(); int32_t result = handleNext(fData->fForwardTable); if (fDictionaryCharCount > 0) { result = checkDictionary(startPos, result, FALSE); } return result; } /** * Advances the iterator backwards, to the last boundary preceding this one. * @return The position of the last boundary position preceding this one. */ int32_t RuleBasedBreakIterator::previous(void) { int32_t result; int32_t startPos; // if we have cached break positions and we're still in the range // covered by them, just move one step backward in the cache if (fCachedBreakPositions != NULL) { if (fPositionInCache > 0) { --fPositionInCache; // If we're at the beginning of the cache, need to reevaluate the // rule status if (fPositionInCache <= 0) { fLastStatusIndexValid = FALSE; } int32_t pos = fCachedBreakPositions[fPositionInCache]; utext_setNativeIndex(fText, pos); return pos; } else { reset(); } } // if we're already sitting at the beginning of the text, return DONE if (fText == NULL || (startPos = current()) == 0) { fLastRuleStatusIndex = 0; fLastStatusIndexValid = TRUE; return BreakIterator::DONE; } if (fData->fSafeRevTable != NULL || fData->fSafeFwdTable != NULL) { result = handlePrevious(fData->fReverseTable); if (fDictionaryCharCount > 0) { result = checkDictionary(result, startPos, TRUE); } return result; } // old rule syntax // set things up. handlePrevious() will back us up to some valid // break position before the current position (we back our internal // iterator up one step to prevent handlePrevious() from returning // the current position), but not necessarily the last one before // where we started int32_t start = current(); (void)UTEXT_PREVIOUS32(fText); int32_t lastResult = handlePrevious(fData->fReverseTable); if (lastResult == UBRK_DONE) { lastResult = 0; utext_setNativeIndex(fText, 0); } result = lastResult; int32_t lastTag = 0; UBool breakTagValid = FALSE; // iterate forward from the known break position until we pass our // starting point. The last break position before the starting // point is our return value for (;;) { result = next(); if (result == BreakIterator::DONE || result >= start) { break; } lastResult = result; lastTag = fLastRuleStatusIndex; breakTagValid = TRUE; } // fLastBreakTag wants to have the value for section of text preceding // the result position that we are to return (in lastResult.) If // the backwards rules overshot and the above loop had to do two or more // next()s to move up to the desired return position, we will have a valid // tag value. But, if handlePrevious() took us to exactly the correct result positon, // we wont have a tag value for that position, which is only set by handleNext(). // set the current iteration position to be the last break position // before where we started, and then return that value utext_setNativeIndex(fText, lastResult); fLastRuleStatusIndex = lastTag; // for use by getRuleStatus() fLastStatusIndexValid = breakTagValid; // No need to check the dictionary; it will have been handled by // next() return lastResult; } /** * Sets the iterator to refer to the first boundary position following * the specified position. * @offset The position from which to begin searching for a break position. * @return The position of the first break after the current position. */ int32_t RuleBasedBreakIterator::following(int32_t offset) { // if we have cached break positions and offset is in the range // covered by them, use them // TODO: could use binary search // TODO: what if offset is outside range, but break is not? if (fCachedBreakPositions != NULL) { if (offset >= fCachedBreakPositions[0] && offset < fCachedBreakPositions[fNumCachedBreakPositions - 1]) { fPositionInCache = 0; // We are guaranteed not to leave the array due to range test above while (offset >= fCachedBreakPositions[fPositionInCache]) { ++fPositionInCache; } int32_t pos = fCachedBreakPositions[fPositionInCache]; utext_setNativeIndex(fText, pos); return pos; } else { reset(); } } // if the offset passed in is already past the end of the text, // just return DONE; if it's before the beginning, return the // text's starting offset fLastRuleStatusIndex = 0; fLastStatusIndexValid = TRUE; if (fText == NULL || offset >= utext_nativeLength(fText)) { last(); return next(); } else if (offset < 0) { return first(); } // otherwise, set our internal iteration position (temporarily) // to the position passed in. If this is the _beginning_ position, // then we can just use next() to get our return value int32_t result = 0; if (fData->fSafeRevTable != NULL) { // new rule syntax utext_setNativeIndex(fText, offset); // move forward one codepoint to prepare for moving back to a // safe point. // this handles offset being between a supplementary character (void)UTEXT_NEXT32(fText); // handlePrevious will move most of the time to < 1 boundary away handlePrevious(fData->fSafeRevTable); int32_t result = next(); while (result <= offset) { result = next(); } return result; } if (fData->fSafeFwdTable != NULL) { // backup plan if forward safe table is not available utext_setNativeIndex(fText, offset); (void)UTEXT_PREVIOUS32(fText); // handle next will give result >= offset handleNext(fData->fSafeFwdTable); // previous will give result 0 or 1 boundary away from offset, // most of the time // we have to int32_t oldresult = previous(); while (oldresult > offset) { int32_t result = previous(); if (result <= offset) { return oldresult; } oldresult = result; } int32_t result = next(); if (result <= offset) { return next(); } return result; } // otherwise, we have to sync up first. Use handlePrevious() to back // up to a known break position before the specified position (if // we can determine that the specified position is a break position, // we don't back up at all). This may or may not be the last break // position at or before our starting position. Advance forward // from here until we've passed the starting position. The position // we stop on will be the first break position after the specified one. // old rule syntax utext_setNativeIndex(fText, offset); if (offset==0 || (offset==1 && utext_getNativeIndex(fText)==0)) { return next(); } result = previous(); while (result != BreakIterator::DONE && result <= offset) { result = next(); } return result; } /** * Sets the iterator to refer to the last boundary position before the * specified position. * @offset The position to begin searching for a break from. * @return The position of the last boundary before the starting position. */ int32_t RuleBasedBreakIterator::preceding(int32_t offset) { // if we have cached break positions and offset is in the range // covered by them, use them if (fCachedBreakPositions != NULL) { // TODO: binary search? // TODO: What if offset is outside range, but break is not? if (offset > fCachedBreakPositions[0] && offset <= fCachedBreakPositions[fNumCachedBreakPositions - 1]) { fPositionInCache = 0; while (fPositionInCache < fNumCachedBreakPositions && offset > fCachedBreakPositions[fPositionInCache]) ++fPositionInCache; --fPositionInCache; // If we're at the beginning of the cache, need to reevaluate the // rule status if (fPositionInCache <= 0) { fLastStatusIndexValid = FALSE; } utext_setNativeIndex(fText, fCachedBreakPositions[fPositionInCache]); return fCachedBreakPositions[fPositionInCache]; } else { reset(); } } // if the offset passed in is already past the end of the text, // just return DONE; if it's before the beginning, return the // text's starting offset if (fText == NULL || offset > utext_nativeLength(fText)) { // return BreakIterator::DONE; return last(); } else if (offset < 0) { return first(); } // if we start by updating the current iteration position to the // position specified by the caller, we can just use previous() // to carry out this operation if (fData->fSafeFwdTable != NULL) { // new rule syntax utext_setNativeIndex(fText, offset); int32_t newOffset = (int32_t)UTEXT_GETNATIVEINDEX(fText); if (newOffset != offset) { // Will come here if specified offset was not a code point boundary AND // the underlying implmentation is using UText, which snaps any non-code-point-boundary // indices to the containing code point. // For breakitereator::preceding only, these non-code-point indices need to be moved // up to refer to the following codepoint. (void)UTEXT_NEXT32(fText); offset = (int32_t)UTEXT_GETNATIVEINDEX(fText); } // TODO: (synwee) would it be better to just check for being in the middle of a surrogate pair, // rather than adjusting the position unconditionally? // (Change would interact with safe rules.) // TODO: change RBBI behavior for off-boundary indices to match that of UText? // affects only preceding(), seems cleaner, but is slightly different. (void)UTEXT_PREVIOUS32(fText); handleNext(fData->fSafeFwdTable); int32_t result = (int32_t)UTEXT_GETNATIVEINDEX(fText); while (result >= offset) { result = previous(); } return result; } if (fData->fSafeRevTable != NULL) { // backup plan if forward safe table is not available // TODO: check whether this path can be discarded // It's probably OK to say that rules must supply both safe tables // if they use safe tables at all. We have certainly never described // to anyone how to work with just one safe table. utext_setNativeIndex(fText, offset); (void)UTEXT_NEXT32(fText); // handle previous will give result <= offset handlePrevious(fData->fSafeRevTable); // next will give result 0 or 1 boundary away from offset, // most of the time // we have to int32_t oldresult = next(); while (oldresult < offset) { int32_t result = next(); if (result >= offset) { return oldresult; } oldresult = result; } int32_t result = previous(); if (result >= offset) { return previous(); } return result; } // old rule syntax utext_setNativeIndex(fText, offset); return previous(); } /** * Returns true if the specfied position is a boundary position. As a side * effect, leaves the iterator pointing to the first boundary position at * or after "offset". * @param offset the offset to check. * @return True if "offset" is a boundary position. */ UBool RuleBasedBreakIterator::isBoundary(int32_t offset) { // the beginning index of the iterator is always a boundary position by definition if (offset == 0) { first(); // For side effects on current position, tag values. return TRUE; } if (offset == (int32_t)utext_nativeLength(fText)) { last(); // For side effects on current position, tag values. return TRUE; } // out-of-range indexes are never boundary positions if (offset < 0) { first(); // For side effects on current position, tag values. return FALSE; } if (offset > utext_nativeLength(fText)) { last(); // For side effects on current position, tag values. return FALSE; } // otherwise, we can use following() on the position before the specified // one and return true if the position we get back is the one the user // specified utext_previous32From(fText, offset); int32_t backOne = (int32_t)UTEXT_GETNATIVEINDEX(fText); UBool result = following(backOne) == offset; return result; } /** * Returns the current iteration position. * @return The current iteration position. */ int32_t RuleBasedBreakIterator::current(void) const { int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(fText); return pos; } //======================================================================= // implementation //======================================================================= // // RBBIRunMode - the state machine runs an extra iteration at the beginning and end // of user text. A variable with this enum type keeps track of where we // are. The state machine only fetches user input while in the RUN mode. // enum RBBIRunMode { RBBI_START, // state machine processing is before first char of input RBBI_RUN, // state machine processing is in the user text RBBI_END // state machine processing is after end of user text. }; //----------------------------------------------------------------------------------- // // handleNext(stateTable) // This method is the actual implementation of the rbbi next() method. // This method initializes the state machine to state 1 // and advances through the text character by character until we reach the end // of the text or the state machine transitions to state 0. We update our return // value every time the state machine passes through an accepting state. // //----------------------------------------------------------------------------------- int32_t RuleBasedBreakIterator::handleNext(const RBBIStateTable *statetable) { int32_t state; uint16_t category = 0; RBBIRunMode mode; RBBIStateTableRow *row; UChar32 c; int32_t lookaheadStatus = 0; int32_t lookaheadTagIdx = 0; int32_t result = 0; int32_t initialPosition = 0; int32_t lookaheadResult = 0; UBool lookAheadHardBreak = (statetable->fFlags & RBBI_LOOKAHEAD_HARD_BREAK) != 0; const char *tableData = statetable->fTableData; uint32_t tableRowLen = statetable->fRowLen; #ifdef RBBI_DEBUG if (fTrace) { RBBIDebugPuts("Handle Next pos char state category"); } #endif // No matter what, handleNext alway correctly sets the break tag value. fLastStatusIndexValid = TRUE; fLastRuleStatusIndex = 0; // if we're already at the end of the text, return DONE. initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText); result = initialPosition; c = UTEXT_NEXT32(fText); if (fData == NULL || c==U_SENTINEL) { return BreakIterator::DONE; } // Set the initial state for the state machine state = START_STATE; row = (RBBIStateTableRow *) //(statetable->fTableData + (statetable->fRowLen * state)); (tableData + tableRowLen * state); mode = RBBI_RUN; if (statetable->fFlags & RBBI_BOF_REQUIRED) { category = 2; mode = RBBI_START; } // loop until we reach the end of the text or transition to state 0 // for (;;) { if (c == U_SENTINEL) { // Reached end of input string. if (mode == RBBI_END) { // We have already run the loop one last time with the // character set to the psueudo {eof} value. Now it is time // to unconditionally bail out. if (lookaheadResult > result) { // We ran off the end of the string with a pending look-ahead match. // Treat this as if the look-ahead condition had been met, and return // the match at the / position from the look-ahead rule. result = lookaheadResult; fLastRuleStatusIndex = lookaheadTagIdx; lookaheadStatus = 0; } break; } // Run the loop one last time with the fake end-of-input character category. mode = RBBI_END; category = 1; } // // Get the char category. An incoming category of 1 or 2 means that // we are preset for doing the beginning or end of input, and // that we shouldn't get a category from an actual text input character. // if (mode == RBBI_RUN) { // look up the current character's character category, which tells us // which column in the state table to look at. // Note: the 16 in UTRIE_GET16 refers to the size of the data being returned, // not the size of the character going in, which is a UChar32. // UTRIE_GET16(&fData->fTrie, c, category); // Check the dictionary bit in the character's category. // Counter is only used by dictionary based iterators (subclasses). // Chars that need to be handled by a dictionary have a flag bit set // in their category values. // if ((category & 0x4000) != 0) { fDictionaryCharCount++; // And off the dictionary flag bit. category &= ~0x4000; } } #ifdef RBBI_DEBUG if (fTrace) { RBBIDebugPrintf(" %4ld ", utext_getNativeIndex(fText)); if (0x20<=c && c<0x7f) { RBBIDebugPrintf("\"%c\" ", c); } else { RBBIDebugPrintf("%5x ", c); } RBBIDebugPrintf("%3d %3d\n", state, category); } #endif // State Transition - move machine to its next state // // Note: fNextState is defined as uint16_t[2], but we are casting // a generated RBBI table to RBBIStateTableRow and some tables // actually have more than 2 categories. U_ASSERT(category<fData->fHeader->fCatCount); state = row->fNextState[category]; /*Not accessing beyond memory*/ row = (RBBIStateTableRow *) // (statetable->fTableData + (statetable->fRowLen * state)); (tableData + tableRowLen * state); if (row->fAccepting == -1) { // Match found, common case. if (mode != RBBI_START) { result = (int32_t)UTEXT_GETNATIVEINDEX(fText); } fLastRuleStatusIndex = row->fTagIdx; // Remember the break status (tag) values. } if (row->fLookAhead != 0) { if (lookaheadStatus != 0 && row->fAccepting == lookaheadStatus) { // Lookahead match is completed. result = lookaheadResult; fLastRuleStatusIndex = lookaheadTagIdx; lookaheadStatus = 0; // TODO: make a standalone hard break in a rule work. if (lookAheadHardBreak) { UTEXT_SETNATIVEINDEX(fText, result); return result; } // Look-ahead completed, but other rules may match further. Continue on // TODO: junk this feature? I don't think it's used anywhwere. goto continueOn; } int32_t r = (int32_t)UTEXT_GETNATIVEINDEX(fText); lookaheadResult = r; lookaheadStatus = row->fLookAhead; lookaheadTagIdx = row->fTagIdx; goto continueOn; } if (row->fAccepting != 0) { // Because this is an accepting state, any in-progress look-ahead match // is no longer relavant. Clear out the pending lookahead status. lookaheadStatus = 0; // clear out any pending look-ahead match. } continueOn: if (state == STOP_STATE) { // This is the normal exit from the lookup state machine. // We have advanced through the string until it is certain that no // longer match is possible, no matter what characters follow. break; } // Advance to the next character. // If this is a beginning-of-input loop iteration, don't advance // the input position. The next iteration will be processing the // first real input character. if (mode == RBBI_RUN) { c = UTEXT_NEXT32(fText); } else { if (mode == RBBI_START) { mode = RBBI_RUN; } } } // The state machine is done. Check whether it found a match... // If the iterator failed to advance in the match engine, force it ahead by one. // (This really indicates a defect in the break rules. They should always match // at least one character.) if (result == initialPosition) { UTEXT_SETNATIVEINDEX(fText, initialPosition); UTEXT_NEXT32(fText); result = (int32_t)UTEXT_GETNATIVEINDEX(fText); } // Leave the iterator at our result position. UTEXT_SETNATIVEINDEX(fText, result); #ifdef RBBI_DEBUG if (fTrace) { RBBIDebugPrintf("result = %d\n\n", result); } #endif return result; } //----------------------------------------------------------------------------------- // // handlePrevious() // // Iterate backwards, according to the logic of the reverse rules. // This version handles the exact style backwards rules. // // The logic of this function is very similar to handleNext(), above. // //----------------------------------------------------------------------------------- int32_t RuleBasedBreakIterator::handlePrevious(const RBBIStateTable *statetable) { int32_t state; uint16_t category = 0; RBBIRunMode mode; RBBIStateTableRow *row; UChar32 c; int32_t lookaheadStatus = 0; int32_t result = 0; int32_t initialPosition = 0; int32_t lookaheadResult = 0; UBool lookAheadHardBreak = (statetable->fFlags & RBBI_LOOKAHEAD_HARD_BREAK) != 0; #ifdef RBBI_DEBUG if (fTrace) { RBBIDebugPuts("Handle Previous pos char state category"); } #endif // handlePrevious() never gets the rule status. // Flag the status as invalid; if the user ever asks for status, we will need // to back up, then re-find the break position using handleNext(), which does // get the status value. fLastStatusIndexValid = FALSE; fLastRuleStatusIndex = 0; // if we're already at the start of the text, return DONE. if (fText == NULL || fData == NULL || UTEXT_GETNATIVEINDEX(fText)==0) { return BreakIterator::DONE; } // Set up the starting char. initialPosition = (int32_t)UTEXT_GETNATIVEINDEX(fText); result = initialPosition; c = UTEXT_PREVIOUS32(fText); // Set the initial state for the state machine state = START_STATE; row = (RBBIStateTableRow *) (statetable->fTableData + (statetable->fRowLen * state)); category = 3; mode = RBBI_RUN; if (statetable->fFlags & RBBI_BOF_REQUIRED) { category = 2; mode = RBBI_START; } // loop until we reach the start of the text or transition to state 0 // for (;;) { if (c == U_SENTINEL) { // Reached end of input string. if (mode == RBBI_END) { // We have already run the loop one last time with the // character set to the psueudo {eof} value. Now it is time // to unconditionally bail out. if (lookaheadResult < result) { // We ran off the end of the string with a pending look-ahead match. // Treat this as if the look-ahead condition had been met, and return // the match at the / position from the look-ahead rule. result = lookaheadResult; lookaheadStatus = 0; } else if (result == initialPosition) { // Ran off start, no match found. // move one index one (towards the start, since we are doing a previous()) UTEXT_SETNATIVEINDEX(fText, initialPosition); (void)UTEXT_PREVIOUS32(fText); // TODO: shouldn't be necessary. We're already at beginning. Check. } break; } // Run the loop one last time with the fake end-of-input character category. mode = RBBI_END; category = 1; } // // Get the char category. An incoming category of 1 or 2 means that // we are preset for doing the beginning or end of input, and // that we shouldn't get a category from an actual text input character. // if (mode == RBBI_RUN) { // look up the current character's character category, which tells us // which column in the state table to look at. // Note: the 16 in UTRIE_GET16 refers to the size of the data being returned, // not the size of the character going in, which is a UChar32. // UTRIE_GET16(&fData->fTrie, c, category); // Check the dictionary bit in the character's category. // Counter is only used by dictionary based iterators (subclasses). // Chars that need to be handled by a dictionary have a flag bit set // in their category values. // if ((category & 0x4000) != 0) { fDictionaryCharCount++; // And off the dictionary flag bit. category &= ~0x4000; } } #ifdef RBBI_DEBUG if (fTrace) { RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(fText)); if (0x20<=c && c<0x7f) { RBBIDebugPrintf("\"%c\" ", c); } else { RBBIDebugPrintf("%5x ", c); } RBBIDebugPrintf("%3d %3d\n", state, category); } #endif // State Transition - move machine to its next state // // Note: fNextState is defined as uint16_t[2], but we are casting // a generated RBBI table to RBBIStateTableRow and some tables // actually have more than 2 categories. U_ASSERT(category<fData->fHeader->fCatCount); state = row->fNextState[category]; /*Not accessing beyond memory*/ row = (RBBIStateTableRow *) (statetable->fTableData + (statetable->fRowLen * state)); if (row->fAccepting == -1) { // Match found, common case. result = (int32_t)UTEXT_GETNATIVEINDEX(fText); } if (row->fLookAhead != 0) { if (lookaheadStatus != 0 && row->fAccepting == lookaheadStatus) { // Lookahead match is completed. result = lookaheadResult; lookaheadStatus = 0; // TODO: make a standalone hard break in a rule work. if (lookAheadHardBreak) { UTEXT_SETNATIVEINDEX(fText, result); return result; } // Look-ahead completed, but other rules may match further. Continue on // TODO: junk this feature? I don't think it's used anywhwere. goto continueOn; } int32_t r = (int32_t)UTEXT_GETNATIVEINDEX(fText); lookaheadResult = r; lookaheadStatus = row->fLookAhead; goto continueOn; } if (row->fAccepting != 0) { // Because this is an accepting state, any in-progress look-ahead match // is no longer relavant. Clear out the pending lookahead status. lookaheadStatus = 0; } continueOn: if (state == STOP_STATE) { // This is the normal exit from the lookup state machine. // We have advanced through the string until it is certain that no // longer match is possible, no matter what characters follow. break; } // Move (backwards) to the next character to process. // If this is a beginning-of-input loop iteration, don't advance // the input position. The next iteration will be processing the // first real input character. if (mode == RBBI_RUN) { c = UTEXT_PREVIOUS32(fText); } else { if (mode == RBBI_START) { mode = RBBI_RUN; } } } // The state machine is done. Check whether it found a match... // If the iterator failed to advance in the match engine, force it ahead by one. // (This really indicates a defect in the break rules. They should always match // at least one character.) if (result == initialPosition) { UTEXT_SETNATIVEINDEX(fText, initialPosition); UTEXT_PREVIOUS32(fText); result = (int32_t)UTEXT_GETNATIVEINDEX(fText); } // Leave the iterator at our result position. UTEXT_SETNATIVEINDEX(fText, result); #ifdef RBBI_DEBUG if (fTrace) { RBBIDebugPrintf("result = %d\n\n", result); } #endif return result; } void RuleBasedBreakIterator::reset() { if (fCachedBreakPositions) { uprv_free(fCachedBreakPositions); } fCachedBreakPositions = NULL; fNumCachedBreakPositions = 0; fDictionaryCharCount = 0; fPositionInCache = 0; } //------------------------------------------------------------------------------- // // getRuleStatus() Return the break rule tag associated with the current // iterator position. If the iterator arrived at its current // position by iterating forwards, the value will have been // cached by the handleNext() function. // // If no cached status value is available, the status is // found by doing a previous() followed by a next(), which // leaves the iterator where it started, and computes the // status while doing the next(). // //------------------------------------------------------------------------------- void RuleBasedBreakIterator::makeRuleStatusValid() { if (fLastStatusIndexValid == FALSE) { // No cached status is available. if (fText == NULL || current() == 0) { // At start of text, or there is no text. Status is always zero. fLastRuleStatusIndex = 0; fLastStatusIndexValid = TRUE; } else { // Not at start of text. Find status the tedious way. int32_t pa = current(); previous(); if (fNumCachedBreakPositions > 0) { reset(); // Blow off the dictionary cache } int32_t pb = next(); if (pa != pb) { // note: the if (pa != pb) test is here only to eliminate warnings for // unused local variables on gcc. Logically, it isn't needed. U_ASSERT(pa == pb); } } } U_ASSERT(fLastRuleStatusIndex >= 0 && fLastRuleStatusIndex < fData->fStatusMaxIdx); } int32_t RuleBasedBreakIterator::getRuleStatus() const { RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this; nonConstThis->makeRuleStatusValid(); // fLastRuleStatusIndex indexes to the start of the appropriate status record // (the number of status values.) // This function returns the last (largest) of the array of status values. int32_t idx = fLastRuleStatusIndex + fData->fRuleStatusTable[fLastRuleStatusIndex]; int32_t tagVal = fData->fRuleStatusTable[idx]; return tagVal; } int32_t RuleBasedBreakIterator::getRuleStatusVec( int32_t *fillInVec, int32_t capacity, UErrorCode &status) { if (U_FAILURE(status)) { return 0; } RuleBasedBreakIterator *nonConstThis = (RuleBasedBreakIterator *)this; nonConstThis->makeRuleStatusValid(); int32_t numVals = fData->fRuleStatusTable[fLastRuleStatusIndex]; int32_t numValsToCopy = numVals; if (numVals > capacity) { status = U_BUFFER_OVERFLOW_ERROR; numValsToCopy = capacity; } int i; for (i=0; i<numValsToCopy; i++) { fillInVec[i] = fData->fRuleStatusTable[fLastRuleStatusIndex + i + 1]; } return numVals; } //------------------------------------------------------------------------------- // // getBinaryRules Access to the compiled form of the rules, // for use by build system tools that save the data // for standard iterator types. // //------------------------------------------------------------------------------- const uint8_t *RuleBasedBreakIterator::getBinaryRules(uint32_t &length) { const uint8_t *retPtr = NULL; length = 0; if (fData != NULL) { retPtr = (const uint8_t *)fData->fHeader; length = fData->fHeader->fLength; } return retPtr; } //------------------------------------------------------------------------------- // // BufferClone TODO: In my (Andy) opinion, this function should be deprecated. // Saving one heap allocation isn't worth the trouble. // Cloning shouldn't be done in tight loops, and // making the clone copy involves other heap operations anyway. // And the application code for correctly dealing with buffer // size problems and the eventual object destruction is ugly. // //------------------------------------------------------------------------------- BreakIterator * RuleBasedBreakIterator::createBufferClone(void *stackBuffer, int32_t &bufferSize, UErrorCode &status) { if (U_FAILURE(status)){ return NULL; } // // If user buffer size is zero this is a preflight operation to // obtain the needed buffer size, allowing for worst case misalignment. // if (bufferSize == 0) { bufferSize = sizeof(RuleBasedBreakIterator) + U_ALIGNMENT_OFFSET_UP(0); return NULL; } // // Check the alignment and size of the user supplied buffer. // Allocate heap memory if the user supplied memory is insufficient. // char *buf = (char *)stackBuffer; uint32_t s = bufferSize; if (stackBuffer == NULL) { s = 0; // Ignore size, force allocation if user didn't give us a buffer. } if (U_ALIGNMENT_OFFSET(stackBuffer) != 0) { uint32_t offsetUp = (uint32_t)U_ALIGNMENT_OFFSET_UP(buf); s -= offsetUp; buf += offsetUp; } if (s < sizeof(RuleBasedBreakIterator)) { // Not enough room in the caller-supplied buffer. // Do a plain-vanilla heap based clone and return that, along with // a warning that the clone was allocated. RuleBasedBreakIterator *clonedBI = new RuleBasedBreakIterator(*this); if (clonedBI == 0) { status = U_MEMORY_ALLOCATION_ERROR; } else { status = U_SAFECLONE_ALLOCATED_WARNING; } return clonedBI; } // // Clone the source BI into the caller-supplied buffer. // RuleBasedBreakIterator *clone = new(buf) RuleBasedBreakIterator(*this); clone->fBufferClone = TRUE; // Flag to prevent deleting storage on close (From C code) return clone; } //------------------------------------------------------------------------------- // // isDictionaryChar Return true if the category lookup for this char // indicates that it is in the set of dictionary lookup // chars. // // This function is intended for use by dictionary based // break iterators. // //------------------------------------------------------------------------------- /*UBool RuleBasedBreakIterator::isDictionaryChar(UChar32 c) { if (fData == NULL) { return FALSE; } uint16_t category; UTRIE_GET16(&fData->fTrie, c, category); return (category & 0x4000) != 0; }*/ //------------------------------------------------------------------------------- // // checkDictionary This function handles all processing of characters in // the "dictionary" set. It will determine the appropriate // course of action, and possibly set up a cache in the // process. // //------------------------------------------------------------------------------- int32_t RuleBasedBreakIterator::checkDictionary(int32_t startPos, int32_t endPos, UBool reverse) { // Reset the old break cache first. uint32_t dictionaryCount = fDictionaryCharCount; reset(); if (dictionaryCount <= 1 || (endPos - startPos) <= 1) { return (reverse ? startPos : endPos); } // Bug 5532. The dictionary code will crash if the input text is UTF-8 // because native indexes are different from UTF-16 indexes. // Temporary hack: skip dictionary lookup for UTF-8 encoded text. // It wont give the right breaks, but it's better than a crash. // // Check the type of the UText by checking its pFuncs field, which // is UText's function dispatch table. It will be the same for all // UTF-8 UTexts and different for any other UText type. // // We have no other type of UText available with non-UTF-16 native indexing. // This whole check will go away once the dictionary code is fixed. static const void *utext_utf8Funcs; if (utext_utf8Funcs == NULL) { // Cache the UTF-8 UText function pointer value. UErrorCode status = U_ZERO_ERROR; UText tempUText = UTEXT_INITIALIZER; utext_openUTF8(&tempUText, NULL, 0, &status); utext_utf8Funcs = tempUText.pFuncs; utext_close(&tempUText); } if (fText->pFuncs == utext_utf8Funcs) { return (reverse ? startPos : endPos); } // Starting from the starting point, scan towards the proposed result, // looking for the first dictionary character (which may be the one // we're on, if we're starting in the middle of a range). utext_setNativeIndex(fText, reverse ? endPos : startPos); if (reverse) { UTEXT_PREVIOUS32(fText); } int32_t rangeStart = startPos; int32_t rangeEnd = endPos; uint16_t category; int32_t current; UErrorCode status = U_ZERO_ERROR; UStack breaks(status); int32_t foundBreakCount = 0; UChar32 c = utext_current32(fText); UTRIE_GET16(&fData->fTrie, c, category); // Is the character we're starting on a dictionary character? If so, we // need to back up to include the entire run; otherwise the results of // the break algorithm will differ depending on where we start. Since // the result is cached and there is typically a non-dictionary break // within a small number of words, there should be little performance impact. if (category & 0x4000) { if (reverse) { do { utext_next32(fText); // TODO: recast to work directly with postincrement. c = utext_current32(fText); UTRIE_GET16(&fData->fTrie, c, category); } while (c != U_SENTINEL && (category & 0x4000)); // Back up to the last dictionary character rangeEnd = (int32_t)UTEXT_GETNATIVEINDEX(fText); if (c == U_SENTINEL) { // c = fText->last32(); // TODO: why was this if needed? c = UTEXT_PREVIOUS32(fText); } else { c = UTEXT_PREVIOUS32(fText); } } else { do { c = UTEXT_PREVIOUS32(fText); UTRIE_GET16(&fData->fTrie, c, category); } while (c != U_SENTINEL && (category & 0x4000)); // Back up to the last dictionary character if (c == U_SENTINEL) { // c = fText->first32(); c = utext_current32(fText); } else { utext_next32(fText); c = utext_current32(fText); } rangeStart = (int32_t)UTEXT_GETNATIVEINDEX(fText);; } UTRIE_GET16(&fData->fTrie, c, category); } // Loop through the text, looking for ranges of dictionary characters. // For each span, find the appropriate break engine, and ask it to find // any breaks within the span. // Note: we always do this in the forward direction, so that the break // cache is built in the right order. if (reverse) { utext_setNativeIndex(fText, rangeStart); c = utext_current32(fText); UTRIE_GET16(&fData->fTrie, c, category); } while(U_SUCCESS(status)) { while((current = (int32_t)UTEXT_GETNATIVEINDEX(fText)) < rangeEnd && (category & 0x4000) == 0) { utext_next32(fText); // TODO: tweak for post-increment operation c = utext_current32(fText); UTRIE_GET16(&fData->fTrie, c, category); } if (current >= rangeEnd) { break; } // We now have a dictionary character. Get the appropriate language object // to deal with it. const LanguageBreakEngine *lbe = getLanguageBreakEngine(c); // Ask the language object if there are any breaks. It will leave the text // pointer on the other side of its range, ready to search for the next one. if (lbe != NULL) { foundBreakCount += lbe->findBreaks(fText, rangeStart, rangeEnd, FALSE, fBreakType, breaks); } // Reload the loop variables for the next go-round c = utext_current32(fText); UTRIE_GET16(&fData->fTrie, c, category); } // If we found breaks, build a new break cache. The first and last entries must // be the original starting and ending position. if (foundBreakCount > 0) { int32_t totalBreaks = foundBreakCount; if (startPos < breaks.elementAti(0)) { totalBreaks += 1; } if (endPos > breaks.peeki()) { totalBreaks += 1; } fCachedBreakPositions = (int32_t *)uprv_malloc(totalBreaks * sizeof(int32_t)); if (fCachedBreakPositions != NULL) { int32_t out = 0; fNumCachedBreakPositions = totalBreaks; if (startPos < breaks.elementAti(0)) { fCachedBreakPositions[out++] = startPos; } for (int32_t i = 0; i < foundBreakCount; ++i) { fCachedBreakPositions[out++] = breaks.elementAti(i); } if (endPos > fCachedBreakPositions[out-1]) { fCachedBreakPositions[out] = endPos; } // If there are breaks, then by definition, we are replacing the original // proposed break by one of the breaks we found. Use following() and // preceding() to do the work. They should never recurse in this case. if (reverse) { return preceding(endPos - 1); } else { return following(startPos); } } // If the allocation failed, just fall through to the "no breaks found" case. } // If we get here, there were no language-based breaks. Set the text pointer // to the original proposed break. utext_setNativeIndex(fText, reverse ? startPos : endPos); return (reverse ? startPos : endPos); } U_NAMESPACE_END // defined in ucln_cmn.h static icu::UStack *gLanguageBreakFactories = NULL; /** * Release all static memory held by breakiterator. */ U_CDECL_BEGIN static UBool U_CALLCONV breakiterator_cleanup_dict(void) { if (gLanguageBreakFactories) { delete gLanguageBreakFactories; gLanguageBreakFactories = NULL; } return TRUE; } U_CDECL_END U_CDECL_BEGIN static void U_CALLCONV _deleteFactory(void *obj) { delete (icu::LanguageBreakFactory *) obj; } U_CDECL_END U_NAMESPACE_BEGIN static const LanguageBreakEngine* getLanguageBreakEngineFromFactory(UChar32 c, int32_t breakType) { UBool needsInit; UErrorCode status = U_ZERO_ERROR; UMTX_CHECK(NULL, (UBool)(gLanguageBreakFactories == NULL), needsInit); if (needsInit) { UStack *factories = new UStack(_deleteFactory, NULL, status); if (factories != NULL && U_SUCCESS(status)) { ICULanguageBreakFactory *builtIn = new ICULanguageBreakFactory(status); factories->push(builtIn, status); #ifdef U_LOCAL_SERVICE_HOOK LanguageBreakFactory *extra = (LanguageBreakFactory *)uprv_svc_hook("languageBreakFactory", &status); if (extra != NULL) { factories->push(extra, status); } #endif } umtx_lock(NULL); if (gLanguageBreakFactories == NULL) { gLanguageBreakFactories = factories; factories = NULL; ucln_common_registerCleanup(UCLN_COMMON_BREAKITERATOR_DICT, breakiterator_cleanup_dict); } umtx_unlock(NULL); delete factories; } if (gLanguageBreakFactories == NULL) { return NULL; } int32_t i = gLanguageBreakFactories->size(); const LanguageBreakEngine *lbe = NULL; while (--i >= 0) { LanguageBreakFactory *factory = (LanguageBreakFactory *)(gLanguageBreakFactories->elementAt(i)); lbe = factory->getEngineFor(c, breakType); if (lbe != NULL) { break; } } return lbe; } //------------------------------------------------------------------------------- // // getLanguageBreakEngine Find an appropriate LanguageBreakEngine for the // the characer c. // //------------------------------------------------------------------------------- const LanguageBreakEngine * RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) { const LanguageBreakEngine *lbe = NULL; UErrorCode status = U_ZERO_ERROR; if (fLanguageBreakEngines == NULL) { fLanguageBreakEngines = new UStack(status); if (fLanguageBreakEngines == NULL || U_FAILURE(status)) { delete fLanguageBreakEngines; fLanguageBreakEngines = 0; return NULL; } } int32_t i = fLanguageBreakEngines->size(); while (--i >= 0) { lbe = (const LanguageBreakEngine *)(fLanguageBreakEngines->elementAt(i)); if (lbe->handles(c, fBreakType)) { return lbe; } } // No existing dictionary took the character. See if a factory wants to // give us a new LanguageBreakEngine for this character. lbe = getLanguageBreakEngineFromFactory(c, fBreakType); // If we got one, use it and push it on our stack. if (lbe != NULL) { fLanguageBreakEngines->push((void *)lbe, status); // Even if we can't remember it, we can keep looking it up, so // return it even if the push fails. return lbe; } // No engine is forthcoming for this character. Add it to the // reject set. Create the reject break engine if needed. if (fUnhandledBreakEngine == NULL) { fUnhandledBreakEngine = new UnhandledEngine(status); if (U_SUCCESS(status) && fUnhandledBreakEngine == NULL) { status = U_MEMORY_ALLOCATION_ERROR; } // Put it last so that scripts for which we have an engine get tried // first. fLanguageBreakEngines->insertElementAt(fUnhandledBreakEngine, 0, status); // If we can't insert it, or creation failed, get rid of it if (U_FAILURE(status)) { delete fUnhandledBreakEngine; fUnhandledBreakEngine = 0; return NULL; } } // Tell the reject engine about the character; at its discretion, it may // add more than just the one character. fUnhandledBreakEngine->handleCharacter(c, fBreakType); return fUnhandledBreakEngine; } /*int32_t RuleBasedBreakIterator::getBreakType() const { return fBreakType; }*/ void RuleBasedBreakIterator::setBreakType(int32_t type) { fBreakType = type; reset(); } U_NAMESPACE_END #endif /* #if !UCONFIG_NO_BREAK_ITERATION */