utf8tst.c   [plain text]

/********************************************************************
 * COPYRIGHT:
 * Copyright (c) 1998-2012, International Business Machines Corporation and
 * others. All Rights Reserved.
 ********************************************************************/
/*
* File utf8tst.c
*
* Modification History:
*
*   Date          Name        Description
*   07/24/2000    Madhu       Creation
*******************************************************************************
*/

#include "unicode/utypes.h"
#include "unicode/utf8.h"
#include "cmemory.h"
#include "cintltst.h"

#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))

/* lenient UTF-8 ------------------------------------------------------------ */

/*
 * Lenient UTF-8 differs from conformant UTF-8 in that it allows surrogate
 * code points with their "natural" encoding.
 * Effectively, this allows a mix of UTF-8 and CESU-8 as well as encodings of
 * single surrogates.
 *
 * This is not conformant with UTF-8.
 *
 * Supplementary code points may be encoded as pairs of 3-byte sequences, but
 * the macros below do not attempt to assemble such pairs.
 */

#define L8_NEXT(s, i, length, c) { \
    (c)=(uint8_t)(s)[(i)++]; \
    if((c)>=0x80) { \
        if(U8_IS_LEAD(c)) { \
            (c)=utf8_nextCharSafeBody((const uint8_t *)s, &(i), (int32_t)(length), c, -2); \
        } else { \
            (c)=U_SENTINEL; \
        } \
    } \
}

#define L8_PREV(s, start, i, c) { \
    (c)=(uint8_t)(s)[--(i)]; \
    if((c)>=0x80) { \
        if((c)<=0xbf) { \
            (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -2); \
        } else { \
            (c)=U_SENTINEL; \
        } \
    } \
}

/* -------------------------------------------------------------------------- */

static void printUChars(const uint8_t *uchars, int16_t len);

static void TestCodeUnitValues(void);
static void TestCharLength(void);
static void TestGetChar(void);
static void TestNextPrevChar(void);
static void TestNulTerminated(void);
static void TestNextPrevNonCharacters(void);
static void TestNextPrevCharUnsafe(void);
static void TestFwdBack(void);
static void TestFwdBackUnsafe(void);
static void TestSetChar(void);
static void TestSetCharUnsafe(void);
static void TestAppendChar(void);
static void TestAppend(void);
static void TestSurrogates(void);

void addUTF8Test(TestNode** root);

void
addUTF8Test(TestNode** root)
{
    addTest(root, &TestCodeUnitValues,          "utf8tst/TestCodeUnitValues");
    addTest(root, &TestCharLength,              "utf8tst/TestCharLength");
    addTest(root, &TestGetChar,                 "utf8tst/TestGetChar");
    addTest(root, &TestNextPrevChar,            "utf8tst/TestNextPrevChar");
    addTest(root, &TestNulTerminated,           "utf8tst/TestNulTerminated");
    addTest(root, &TestNextPrevNonCharacters,   "utf8tst/TestNextPrevNonCharacters");
    addTest(root, &TestNextPrevCharUnsafe,      "utf8tst/TestNextPrevCharUnsafe");
    addTest(root, &TestFwdBack,                 "utf8tst/TestFwdBack");
    addTest(root, &TestFwdBackUnsafe,           "utf8tst/TestFwdBackUnsafe");
    addTest(root, &TestSetChar,                 "utf8tst/TestSetChar");
    addTest(root, &TestSetCharUnsafe,           "utf8tst/TestSetCharUnsafe");
    addTest(root, &TestAppendChar,              "utf8tst/TestAppendChar");
    addTest(root, &TestAppend,                  "utf8tst/TestAppend");
    addTest(root, &TestSurrogates,              "utf8tst/TestSurrogates");
}

static void TestCodeUnitValues()
{
    static const uint8_t codeunit[]={0x00, 0x65, 0x7e, 0x7f, 0xc0, 0xc4, 0xf0, 0xfd, 0x80, 0x81, 0xbc, 0xbe,};

    int16_t i;
    for(i=0; i<LENGTHOF(codeunit); i++){
        uint8_t c=codeunit[i];
        log_verbose("Testing code unit value of %x\n", c);
        if(i<4){
            if(!UTF8_IS_SINGLE(c) || UTF8_IS_LEAD(c) || UTF8_IS_TRAIL(c) || !U8_IS_SINGLE(c) || U8_IS_LEAD(c) || U8_IS_TRAIL(c)){
                log_err("ERROR: 0x%02x is a single byte but results in single: %c lead: %c trail: %c\n",
                    c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n');
            }
        } else if(i< 8){
            if(!UTF8_IS_LEAD(c) || UTF8_IS_SINGLE(c) || UTF8_IS_TRAIL(c) || !U8_IS_LEAD(c) || U8_IS_SINGLE(c) || U8_IS_TRAIL(c)){
                log_err("ERROR: 0x%02x is a lead byte but results in single: %c lead: %c trail: %c\n",
                    c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n');
            }
        } else if(i< 12){
            if(!UTF8_IS_TRAIL(c) || UTF8_IS_SINGLE(c) || UTF8_IS_LEAD(c) || !U8_IS_TRAIL(c) || U8_IS_SINGLE(c) || U8_IS_LEAD(c)){
                log_err("ERROR: 0x%02x is a trail byte but results in single: %c lead: %c trail: %c\n",
                    c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n');
            }
        }
    }
}

static void TestCharLength()
{
    static const uint32_t codepoint[]={
        1, 0x0061,
        1, 0x007f,
        2, 0x016f,
        2, 0x07ff,
        3, 0x0865,
        3, 0x20ac,
        4, 0x20402,
        4, 0x23456,
        4, 0x24506,
        4, 0x20402,
        4, 0x10402,
        3, 0xd7ff,
        3, 0xe000,

    };

    int16_t i;
    UBool multiple;
    for(i=0; i<LENGTHOF(codepoint); i=(int16_t)(i+2)){
        UChar32 c=codepoint[i+1];
        if(UTF8_CHAR_LENGTH(c) != (uint16_t)codepoint[i] || U8_LENGTH(c) != (uint16_t)codepoint[i]){
              log_err("The no: of code units for %lx:- Expected: %d Got: %d\n", c, codepoint[i], UTF8_CHAR_LENGTH(c));
        }else{
              log_verbose("The no: of code units for %lx is %d\n",c, UTF8_CHAR_LENGTH(c));
        }
        multiple=(UBool)(codepoint[i] == 1 ? FALSE : TRUE);
        if(UTF8_NEED_MULTIPLE_UCHAR(c) != multiple){
              log_err("ERROR: UTF8_NEED_MULTIPLE_UCHAR failed for %lx\n", c);
        }
    }
}

static void TestGetChar()
{
    static const uint8_t input[]={
    /*  code unit,*/
        0x61,
        0x7f,
        0xe4,
        0xba,
        0x8c,
        0xF0,
        0x90,
        0x90,
        0x81,
        0xc0,
        0x65,
        0x31,
        0x9a,
        0xc9
    };
    static const UChar32 result[]={
    /*  codepoint-unsafe, codepoint-safe(not strict)  codepoint-safe(strict) */
        0x61,             0x61,                       0x61,
        0x7f,             0x7f,                       0x7f,
        0x4e8c,           0x4e8c,                     0x4e8c,
        0x4e8c,           0x4e8c,                     0x4e8c ,
        0x4e8c,           0x4e8c,                     0x4e8c,
        0x10401,          0x10401,                    0x10401 ,
        0x10401,          0x10401,                    0x10401 ,
        0x10401,          0x10401,                    0x10401 ,
        0x10401,          0x10401,                    0x10401,
        0x25,             UTF8_ERROR_VALUE_1,         UTF8_ERROR_VALUE_1,
        0x65,             0x65,                       0x65,
        0x31,             0x31,                       0x31,
        0x31,             UTF8_ERROR_VALUE_1,         UTF8_ERROR_VALUE_1,
        0x240,            UTF8_ERROR_VALUE_1,         UTF8_ERROR_VALUE_1
    };
    uint16_t i=0;
    UChar32 c, expected;
    uint32_t offset=0;

    for(offset=0; offset<sizeof(input); offset++) {
        if (offset < sizeof(input) - 1) {
            UTF8_GET_CHAR_UNSAFE(input, offset, c);
            if(c != result[i]){
                log_err("ERROR: UTF8_GET_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i], c);

            }

            U8_GET_UNSAFE(input, offset, c);
            if(c != result[i]){
                log_err("ERROR: U8_GET_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i], c);

            }
        }

        UTF8_GET_CHAR_SAFE(input, 0, offset, sizeof(input), c, FALSE);
        expected=result[i+1];
        if(c != expected){
            log_err("ERROR: UTF8_GET_CHAR_SAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

        U8_GET(input, 0, offset, sizeof(input), c);
        if(UTF_IS_ERROR(expected)) { expected=U_SENTINEL; }
        if(c != expected){
            log_err("ERROR: U8_GET failed for offset=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

        U8_GET_OR_FFFD(input, 0, offset, sizeof(input), c);
        if(expected<0) { expected=0xfffd; }
        if(c != expected){
            log_err("ERROR: U8_GET_OR_FFFD failed for offset=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

        UTF8_GET_CHAR_SAFE(input, 0, offset, sizeof(input), c, TRUE);
        if(c != result[i+2]){
            log_err("ERROR: UTF8_GET_CHAR_SAFE(strict) failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i+2], c);
        }

        i=(uint16_t)(i+3);
    }
}

static void TestNextPrevChar() {
    static const uint8_t input[]={0x61, 0xf0, 0x90, 0x90, 0x81, 0xc0, 0x80, 0xfd, 0xbe, 0xc2, 0x61, 0x81, 0x90, 0x90, 0xf0, 0x00};
    static const UChar32 result[]={
    /*  next_unsafe    next_safe_ns        next_safe_s          prev_unsafe   prev_safe_ns        prev_safe_s */
        0x0061,        0x0061,             0x0061,              0x0000,       0x0000,             0x0000,
        0x10401,       0x10401,            0x10401,             0xf0,         UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
        0x90,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0x2841410,    UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
        0x90,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0xa1050,      UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
        0x81,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0x2841,       UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
        0x00,          UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2,  0x61,         0x61,               0x61,
        0x80,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0xc2,         UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
        0xfd,          UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2,  0x77e,        UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2,
        0xbe,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0xfd,         UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
        0xa1,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0x00,         UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2,
        0x61,          0x61,               0x61,                0xc0,         UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
        0x81,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0x10401,      0x10401,            0x10401,
        0x90,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0x410,        UTF_ERROR_VALUE,    UTF_ERROR_VALUE,
        0x90,          UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0x410,        UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2,
        0x0840,        UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,  0xf0,         UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
        0x0000,        0x0000,             0x0000,              0x0061,       0x0061,             0x0061
    };
    static const int32_t movedOffset[]={
    /*  next_unsafe   next_safe_ns next_safe_s       prev_unsafe   prev_safe_ns      prev_safe_s */
        1,            1,           1,                15,           15,               15,
        5,            5,           5,                14,           14 ,              14,
        3,            3,           3,                9,            13,               13,
        4,            4,           4,                9,            12,               12,
        5,            5,           5,                9,            11,               11,
        7,            7,           7,                10,           10,               10,
        7,            7,           7,                9,            9,                9,
        8,            9,           9,                7,            7,                7,
        9,            9,           9,                7,            7,                7,
        11,           10,          10,               5,            5,                5,
        11,           11,          11,               5,            5,                5,
        12,           12,          12,               1,            1,                1,
        13,           13,          13,               1,            1,                1,
        14,           14,          14,               1,            1,                1,
        14,           15,          15,               1,            1,                1,
        14,           16,          16,               0,            0,                0,
    };
    /* TODO: remove unused columns for next_unsafe & prev_unsafe, and adjust the test code */

    UChar32 c, expected;
    uint32_t i=0;
    uint32_t offset=0;
    int32_t setOffset=0;
    for(offset=0; offset<sizeof(input); offset++){
         setOffset=offset;
         UTF8_NEXT_CHAR_SAFE(input, setOffset, sizeof(input), c, FALSE);
         if(setOffset != movedOffset[i+1]){
             log_err("ERROR: UTF8_NEXT_CHAR_SAFE failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
                 offset, movedOffset[i+1], setOffset);
         }
        expected=result[i+1];
        if(c != expected){
            log_err("ERROR: UTF8_NEXT_CHAR_SAFE failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

         setOffset=offset;
         U8_NEXT(input, setOffset, sizeof(input), c);
         if(setOffset != movedOffset[i+1]){
             log_err("ERROR: U8_NEXT failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
                 offset, movedOffset[i+1], setOffset);
         }
        if(UTF_IS_ERROR(expected)) { expected=U_SENTINEL; }
        if(c != expected){
            log_err("ERROR: U8_NEXT failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

        setOffset=offset;
        U8_NEXT_OR_FFFD(input, setOffset, sizeof(input), c);
        if(setOffset != movedOffset[i+1]){
            log_err("ERROR: U8_NEXT_OR_FFFD failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
                offset, movedOffset[i+1], setOffset);
        }
        if(expected<0) { expected=0xfffd; }
        if(c != expected){
            log_err("ERROR: U8_NEXT_OR_FFFD failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

         setOffset=offset;
         UTF8_NEXT_CHAR_SAFE(input, setOffset, sizeof(input), c, TRUE);
         if(setOffset != movedOffset[i+1]){
             log_err("ERROR: UTF8_NEXT_CHAR_SAFE(strict) failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
                 offset, movedOffset[i+2], setOffset);
         }
         if(c != result[i+2]){
             log_err("ERROR: UTF8_NEXT_CHAR_SAFE(strict) failed for input=%ld. Expected:%lx Got:%lx\n", offset, result[i+2], c);
         }

         i=i+6;
    }

    i=0;
    for(offset=sizeof(input); offset > 0; --offset){
         setOffset=offset;
         UTF8_PREV_CHAR_SAFE(input, 0, setOffset, c, FALSE);
         if(setOffset != movedOffset[i+4]){
             log_err("ERROR: UTF8_PREV_CHAR_SAFE failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
                 offset, movedOffset[i+4], setOffset);
         }
        expected=result[i+4];
        if(c != expected){
            log_err("ERROR: UTF8_PREV_CHAR_SAFE failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

         setOffset=offset;
         U8_PREV(input, 0, setOffset, c);
         if(setOffset != movedOffset[i+4]){
             log_err("ERROR: U8_PREV failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
                 offset, movedOffset[i+4], setOffset);
         }
        if(UTF_IS_ERROR(expected)) { expected=U_SENTINEL; }
        if(c != expected){
            log_err("ERROR: U8_PREV failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

        setOffset=offset;
        U8_PREV_OR_FFFD(input, 0, setOffset, c);
        if(setOffset != movedOffset[i+4]){
            log_err("ERROR: U8_PREV_OR_FFFD failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
                offset, movedOffset[i+4], setOffset);
        }
        if(expected<0) { expected=0xfffd; }
        if(c != expected){
            log_err("ERROR: U8_PREV_OR_FFFD failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c);
        }

         setOffset=offset;
         UTF8_PREV_CHAR_SAFE(input, 0,  setOffset, c, TRUE);
         if(setOffset != movedOffset[i+5]){
             log_err("ERROR: UTF8_PREV_CHAR_SAFE(strict) failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
                 offset, movedOffset[i+5], setOffset);
         }
         if(c != result[i+5]){
             log_err("ERROR: UTF8_PREV_CHAR_SAFE(strict) failed for input=%ld. Expected:%lx Got:%lx\n", offset, result[i+5], c);
         }

         i=i+6;
    }
}

/* keep this in sync with utf16tst.c's TestNulTerminated() */
static void TestNulTerminated() {
    static const uint8_t input[]={
        /*  0 */  0x61,
        /*  1 */  0xf0, 0x90, 0x90, 0x81,
        /*  5 */  0xc0, 0x80,
        /*  7 */  0xdf, 0x80,
        /*  9 */  0xc2,
        /* 10 */  0x62,
        /* 11 */  0xfd, 0xbe,
        /* 13 */  0xe0, 0xa0, 0x80,
        /* 16 */  0xe2, 0x82, 0xac,
        /* 19 */  0xf0, 0x90, 0x90,
        /* 22 */  0x00
        /* 23 */
    };
    static const UChar32 result[]={
        0x61,
        0x10401,
        U_SENTINEL,
        0x7c0,
        U_SENTINEL,
        0x62,
        U_SENTINEL,
        0x800,
        0x20ac,
        U_SENTINEL,
        0
    };

    UChar32 c, c2, expected;
    int32_t i0, i=0, j, k, expectedIndex;
    int32_t cpIndex=0;
    do {
        i0=i;
        U8_NEXT(input, i, -1, c);
        expected=result[cpIndex];
        if(c!=expected) {
            log_err("U8_NEXT(from %d)=U+%04x != U+%04x\n", i0, c, expected);
        }
        j=i0;
        U8_NEXT_OR_FFFD(input, j, -1, c);
        if(expected<0) { expected=0xfffd; }
        if(c!=expected) {
            log_err("U8_NEXT_OR_FFFD(from %d)=U+%04x != U+%04x\n", i0, c, expected);
        }
        if(j!=i) {
            log_err("U8_NEXT_OR_FFFD() moved to index %d but U8_NEXT() moved to %d\n", j, i);
        }
        j=i0;
        U8_FWD_1(input, j, -1);
        if(j!=i) {
            log_err("U8_FWD_1() moved to index %d but U8_NEXT() moved to %d\n", j, i);
        }
        ++cpIndex;
        /*
         * Move by this many code points from the start.
         * U8_FWD_N() stops at the end of the string, that is, at the NUL if necessary.
         */
        expectedIndex= (c==0) ? i-1 : i;
        k=0;
        U8_FWD_N(input, k, -1, cpIndex);
        if(k!=expectedIndex) {
            log_err("U8_FWD_N(code points from 0) moved to index %d but expected %d\n", k, expectedIndex);
        }
    } while(c!=0);

    i=0;
    do {
        j=i0=i;
        U8_NEXT(input, i, -1, c);
        do {
            U8_GET(input, 0, j, -1, c2);
            if(c2!=c) {
                log_err("U8_NEXT(from %d)=U+%04x != U+%04x=U8_GET(at %d)\n", i0, c, c2, j);
            }
            U8_GET_OR_FFFD(input, 0, j, -1, c2);
            expected= (c>=0) ? c : 0xfffd;
            if(c2!=expected) {
                log_err("U8_NEXT_OR_FFFD(from %d)=U+%04x != U+%04x=U8_GET_OR_FFFD(at %d)\n", i0, expected, c2, j);
            }
            /* U8_SET_CP_LIMIT moves from a non-lead byte to the limit of the code point */
            k=j+1;
            U8_SET_CP_LIMIT(input, 0, k, -1);
            if(k!=i) {
                log_err("U8_NEXT() moved to %d but U8_SET_CP_LIMIT(%d) moved to %d\n", i, j+1, k);
            }
        } while(++j<i);
    } while(c!=0);
}

static void TestNextPrevNonCharacters() {
    /* test non-characters */
    static const uint8_t nonChars[]={
        0xef, 0xb7, 0x90,       /* U+fdd0 */
        0xef, 0xbf, 0xbf,       /* U+feff */
        0xf0, 0x9f, 0xbf, 0xbe, /* U+1fffe */
        0xf0, 0xbf, 0xbf, 0xbf, /* U+3ffff */
        0xf4, 0x8f, 0xbf, 0xbe  /* U+10fffe */
    };

    UChar32 ch;
    int32_t idx;

    for(idx=0; idx<(int32_t)sizeof(nonChars);) {
        U8_NEXT(nonChars, idx, sizeof(nonChars), ch);
        if(!U_IS_UNICODE_NONCHAR(ch)) {
            log_err("U8_NEXT(before %d) failed to read a non-character\n", idx);
        }
    }
    for(idx=(int32_t)sizeof(nonChars); idx>0;) {
        U8_PREV(nonChars, 0, idx, ch);
        if(!U_IS_UNICODE_NONCHAR(ch)) {
            log_err("U8_PREV(at %d) failed to read a non-character\n", idx);
        }
    }
}

static void TestNextPrevCharUnsafe() {
    /*
     * Use a (mostly) well-formed UTF-8 string and test at code point boundaries.
     * The behavior of _UNSAFE macros for ill-formed strings is undefined.
     */
    static const uint8_t input[]={
        0x61,
        0xf0, 0x90, 0x90, 0x81,
        0xc0, 0x80,  /* non-shortest form */
        0xe2, 0x82, 0xac,
        0xc2, 0xa1,
        0xf4, 0x8f, 0xbf, 0xbf,
        0x00
    };
    static const UChar32 codePoints[]={
        0x61,
        0x10401,
        0,
        0x20ac,
        0xa1,
        0x10ffff,
        0
    };

    UChar32 c;
    int32_t i;
    uint32_t offset;
    for(i=0, offset=0; offset<sizeof(input); ++i) {
        UTF8_NEXT_CHAR_UNSAFE(input, offset, c);
        if(c != codePoints[i]){
            log_err("ERROR: UTF8_NEXT_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n",
                    offset, codePoints[i], c);
        }
    }
    for(i=0, offset=0; offset<sizeof(input); ++i) {
        U8_NEXT_UNSAFE(input, offset, c);
        if(c != codePoints[i]){
            log_err("ERROR: U8_NEXT_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n",
                    offset, codePoints[i], c);
        }
    }

    for(i=LENGTHOF(codePoints)-1, offset=sizeof(input); offset > 0; --i){
         UTF8_PREV_CHAR_UNSAFE(input, offset, c);
         if(c != codePoints[i]){
             log_err("ERROR: UTF8_PREV_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n",
                     offset, codePoints[i], c);
         }
    }
    for(i=LENGTHOF(codePoints)-1, offset=sizeof(input); offset > 0; --i){
         U8_PREV_UNSAFE(input, offset, c);
         if(c != codePoints[i]){
             log_err("ERROR: U8_PREV_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n",
                     offset, codePoints[i], c);
         }
    }
}

static void TestFwdBack() {
    static const uint8_t input[]={0x61, 0xF0, 0x90, 0x90, 0x81, 0xff, 0x62, 0xc0, 0x80, 0x7f, 0x8f, 0xc0, 0x63, 0x81, 0x90, 0x90, 0xF0, 0x00};
    static const uint16_t fwd_safe[]   ={1, 5, 6, 7, 9, 10, 11,  12, 13, 14, 15, 16, 17, 18};
    static const uint16_t back_safe[]  ={17, 16, 15, 14, 13, 12, 11, 10, 9, 7, 6, 5, 1, 0};

    static const uint16_t Nvalue[]= {0, 1, 2, 3, 1, 2, 1, 5};
    static const uint16_t fwd_N_safe[]   ={0, 1, 6, 10, 11, 13, 14, 18}; /*safe macro keeps it at the end of the string */
    static const uint16_t back_N_safe[]  ={18, 17, 15, 12, 11, 9, 7, 0};

    uint32_t offsafe=0;

    uint32_t i=0;
    while(offsafe < sizeof(input)){
        UTF8_FWD_1_SAFE(input, offsafe, sizeof(input));
        if(offsafe != fwd_safe[i]){
            log_err("ERROR: Forward_safe offset expected:%d, Got:%d\n", fwd_safe[i], offsafe);
        }
        i++;
    }

    i=0;
    while(offsafe < sizeof(input)){
        U8_FWD_1(input, offsafe, sizeof(input));
        if(offsafe != fwd_safe[i]){
            log_err("ERROR: U8_FWD_1 offset expected:%d, Got:%d\n", fwd_safe[i], offsafe);
        }
        i++;
    }

    i=0;
    offsafe=sizeof(input);
    while(offsafe > 0){
        UTF8_BACK_1_SAFE(input, 0,  offsafe);
        if(offsafe != back_safe[i]){
            log_err("ERROR: Backward_safe offset expected:%d, Got:%d\n", back_safe[i], offsafe);
        }
        i++;
    }

    i=0;
    offsafe=sizeof(input);
    while(offsafe > 0){
        U8_BACK_1(input, 0,  offsafe);
        if(offsafe != back_safe[i]){
            log_err("ERROR: U8_BACK_1 offset expected:%d, Got:%d\n", back_safe[i], offsafe);
        }
        i++;
    }

    offsafe=0;
    for(i=0; i<LENGTHOF(Nvalue); i++){
        UTF8_FWD_N_SAFE(input, offsafe, sizeof(input), Nvalue[i]);
        if(offsafe != fwd_N_safe[i]){
            log_err("ERROR: Forward_N_safe offset=%d expected:%d, Got:%d\n", i, fwd_N_safe[i], offsafe);
        }

    }

    offsafe=0;
    for(i=0; i<LENGTHOF(Nvalue); i++){
        U8_FWD_N(input, offsafe, sizeof(input), Nvalue[i]);
        if(offsafe != fwd_N_safe[i]){
            log_err("ERROR: U8_FWD_N offset=%d expected:%d, Got:%d\n", i, fwd_N_safe[i], offsafe);
        }

    }

    offsafe=sizeof(input);
    for(i=0; i<LENGTHOF(Nvalue); i++){
        UTF8_BACK_N_SAFE(input, 0, offsafe, Nvalue[i]);
        if(offsafe != back_N_safe[i]){
            log_err("ERROR: backward_N_safe offset=%d expected:%d, Got:%ld\n", i, back_N_safe[i], offsafe);
        }
    }

    offsafe=sizeof(input);
    for(i=0; i<LENGTHOF(Nvalue); i++){
        U8_BACK_N(input, 0, offsafe, Nvalue[i]);
        if(offsafe != back_N_safe[i]){
            log_err("ERROR: U8_BACK_N offset=%d expected:%d, Got:%ld\n", i, back_N_safe[i], offsafe);
        }
    }
}

static void TestFwdBackUnsafe() {
    /*
     * Use a (mostly) well-formed UTF-8 string and test at code point boundaries.
     * The behavior of _UNSAFE macros for ill-formed strings is undefined.
     */
    static const uint8_t input[]={
        0x61,
        0xf0, 0x90, 0x90, 0x81,
        0xc0, 0x80,  /* non-shortest form */
        0xe2, 0x82, 0xac,
        0xc2, 0xa1,
        0xf4, 0x8f, 0xbf, 0xbf,
        0x00
    };
    static const int8_t boundaries[]={ 0, 1, 5, 7, 10, 12, 16, 17 };

    int32_t offset;
    int32_t i;
    for(i=1, offset=0; offset<LENGTHOF(input); ++i) {
        UTF8_FWD_1_UNSAFE(input, offset);
        if(offset != boundaries[i]){
            log_err("ERROR: UTF8_FWD_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
        }
    }
    for(i=1, offset=0; offset<LENGTHOF(input); ++i) {
        U8_FWD_1_UNSAFE(input, offset);
        if(offset != boundaries[i]){
            log_err("ERROR: U8_FWD_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
        }
    }

    for(i=LENGTHOF(boundaries)-2, offset=LENGTHOF(input); offset>0; --i) {
        UTF8_BACK_1_UNSAFE(input, offset);
        if(offset != boundaries[i]){
            log_err("ERROR: UTF8_BACK_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
        }
    }
    for(i=LENGTHOF(boundaries)-2, offset=LENGTHOF(input); offset>0; --i) {
        U8_BACK_1_UNSAFE(input, offset);
        if(offset != boundaries[i]){
            log_err("ERROR: U8_BACK_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
        }
    }

    for(i=0; i<LENGTHOF(boundaries); ++i) {
        offset=0;
        UTF8_FWD_N_UNSAFE(input, offset, i);
        if(offset != boundaries[i]) {
            log_err("ERROR: UTF8_FWD_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
        }
    }
    for(i=0; i<LENGTHOF(boundaries); ++i) {
        offset=0;
        U8_FWD_N_UNSAFE(input, offset, i);
        if(offset != boundaries[i]) {
            log_err("ERROR: U8_FWD_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
        }
    }

    for(i=0; i<LENGTHOF(boundaries); ++i) {
        int32_t j=LENGTHOF(boundaries)-1-i;
        offset=LENGTHOF(input);
        UTF8_BACK_N_UNSAFE(input, offset, i);
        if(offset != boundaries[j]) {
            log_err("ERROR: UTF8_BACK_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[j], offset);
        }
    }
    for(i=0; i<LENGTHOF(boundaries); ++i) {
        int32_t j=LENGTHOF(boundaries)-1-i;
        offset=LENGTHOF(input);
        U8_BACK_N_UNSAFE(input, offset, i);
        if(offset != boundaries[j]) {
            log_err("ERROR: U8_BACK_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[j], offset);
        }
    }
}

static void TestSetChar() {
    static const uint8_t input[]
        = {0x61, 0xe4, 0xba, 0x8c, 0x7f, 0xfe, 0x62, 0xc5, 0x7f, 0x61, 0x80, 0x80, 0xe0, 0x00 };
    static const int16_t start_safe[]
        = {0,    1,    1,    1,    4,    5,    6,    7,    8,    9,    10,   11,   12,   13,  14 };
    static const int16_t limit_safe[]
        = {0,    1,    4,    4,    4,    5,    6,    7,    8,    9,    10,   11,   12,   13,  14 };

    uint32_t i=0;
    int32_t offset=0, setOffset=0;
    for(offset=0; offset<=LENGTHOF(input); offset++){
        if (offset<LENGTHOF(input)){
            setOffset=offset;
            UTF8_SET_CHAR_START_SAFE(input, 0, setOffset);
            if(setOffset != start_safe[i]){
                log_err("ERROR: UTF8_SET_CHAR_START_SAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_safe[i], setOffset);
            }

            setOffset=offset;
            U8_SET_CP_START(input, 0, setOffset);
            if(setOffset != start_safe[i]){
                log_err("ERROR: U8_SET_CP_START failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_safe[i], setOffset);
            }
        }

        setOffset=offset;
        UTF8_SET_CHAR_LIMIT_SAFE(input,0, setOffset, sizeof(input));
        if(setOffset != limit_safe[i]){
            log_err("ERROR: UTF8_SET_CHAR_LIMIT_SAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_safe[i], setOffset);
        }

        setOffset=offset;
        U8_SET_CP_LIMIT(input,0, setOffset, sizeof(input));
        if(setOffset != limit_safe[i]){
            log_err("ERROR: U8_SET_CP_LIMIT failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_safe[i], setOffset);
        }

        i++;
    }
}

static void TestSetCharUnsafe() {
    static const uint8_t input[]
        = {0x61, 0xe4, 0xba, 0x8c, 0x7f, 0x2e, 0x62, 0xc5, 0x7f, 0x61, 0x80, 0x80, 0xe0, 0x80, 0x80, 0x00 };
    static const int16_t start_unsafe[]
        = {0,    1,    1,    1,    4,    5,    6,    7,    8,    9,    9,    9,    12,   12,   12,   15 };
    static const int16_t limit_unsafe[]
        = {0,    1,    4,    4,    4,    5,    6,    7,    9,    9,    10,   10,   10,   15,   15,   15,   16 };

    uint32_t i=0;
    int32_t offset=0, setOffset=0;
    for(offset=0; offset<=LENGTHOF(input); offset++){
        if (offset<LENGTHOF(input)){
            setOffset=offset;
            UTF8_SET_CHAR_START_UNSAFE(input, setOffset);
            if(setOffset != start_unsafe[i]){
                log_err("ERROR: UTF8_SET_CHAR_START_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_unsafe[i], setOffset);
            }

            setOffset=offset;
            U8_SET_CP_START_UNSAFE(input, setOffset);
            if(setOffset != start_unsafe[i]){
                log_err("ERROR: U8_SET_CP_START_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_unsafe[i], setOffset);
            }
        }

        if (offset != 0) { /* Can't have it go off the end of the array */
            setOffset=offset;
            UTF8_SET_CHAR_LIMIT_UNSAFE(input, setOffset);
            if(setOffset != limit_unsafe[i]){
                log_err("ERROR: UTF8_SET_CHAR_LIMIT_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_unsafe[i], setOffset);
            }

            setOffset=offset;
            U8_SET_CP_LIMIT_UNSAFE(input, setOffset);
            if(setOffset != limit_unsafe[i]){
                log_err("ERROR: U8_SET_CP_LIMIT_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_unsafe[i], setOffset);
            }
        }

        i++;
    }
}

static void TestAppendChar(){
    static const uint8_t s[11]={0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00};
    static const uint32_t test[]={
    /*  append-position(unsafe),  CHAR to be appended */
        0,                        0x10401,
        2,                        0x0028,
        2,                        0x007f,
        3,                        0xd801,
        1,                        0x20402,
        8,                        0x10401,
        5,                        0xc0,
        5,                        0xc1,
        5,                        0xfd,
        6,                        0x80,
        6,                        0x81,
        6,                        0xbf,
        7,                        0xfe,

    /*  append-position(safe),    CHAR to be appended */
        0,                        0x10401,
        2,                        0x0028,
        3,                        0x7f,
        3,                        0xd801,   /* illegal for UTF-8 starting with Unicode 3.2 */
        1,                        0x20402,
        9,                        0x10401,
        5,                        0xc0,
        5,                        0xc1,
        5,                        0xfd,
        6,                        0x80,
        6,                        0x81,
        6,                        0xbf,
        7,                        0xfe,

    };
    static const uint16_t movedOffset[]={
    /* offset-moved-to(unsafe) */
          4,              /*for append-pos: 0 , CHAR 0x10401*/
          3,
          3,
          6,
          5,
          12,
          7,
          7,
          7,
          8,
          8,
          8,
          9,

    /* offset-moved-to(safe) */
          4,              /*for append-pos: 0, CHAR  0x10401*/
          3,
          4,
          6,
          5,
          11,
          7,
          7,
          7,
          8,
          8,
          8,
          9,

    };

    static const uint8_t result[][11]={
        /*unsafe*/
        {0xF0, 0x90, 0x90, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x28, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x7f, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0xed, 0xa0, 0x81, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0xF0, 0xa0, 0x90, 0x82, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0xF0, 0x90, 0x90},

        {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x80, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x81, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0xbd, 0x68, 0x69, 0x6a, 0x00},

        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x80, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x81, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0xbf, 0x69, 0x6a, 0x00},

        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0xc3, 0xbe, 0x6a, 0x00},
        /*safe*/
        {0xF0, 0x90, 0x90, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x28, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x7f, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0xef, 0xbf, 0xbf, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0xF0, 0xa0, 0x90, 0x82, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xc2, 0x9f}, /*gets UTF8_ERROR_VALUE_2 which takes 2 bytes 0xc0, 0x9f*/

        {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x80, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x81, 0x68, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0xbd, 0x68, 0x69, 0x6a, 0x00},

        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x80, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x81, 0x69, 0x6a, 0x00},
        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0xbf, 0x69, 0x6a, 0x00},

        {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0xc3, 0xbe, 0x6a, 0x00},

    };
    uint16_t i, count=0;
    uint8_t str[12];
    uint32_t offset;
/*    UChar32 c=0;*/
    uint16_t size=LENGTHOF(s);
    for(i=0; i<LENGTHOF(test); i=(uint16_t)(i+2)){
        uprv_memcpy(str, s, size);
        offset=test[i];
        if(count<13){
            UTF8_APPEND_CHAR_UNSAFE(str, offset, test[i+1]);
            if(offset != movedOffset[count]){
                log_err("ERROR: UTF8_APPEND_CHAR_UNSAFE failed to move the offset correctly for count=%d.\nExpectedOffset=%d  currentOffset=%d\n",
                    count, movedOffset[count], offset);

            }
            if(uprv_memcmp(str, result[count], size) !=0){
                log_err("ERROR: UTF8_APPEND_CHAR_UNSAFE failed for count=%d. \nExpected:", count);
                printUChars(result[count], size);
                log_err("\nGot:      ");
                printUChars(str, size);
                log_err("\n");
            }
        }else{
            UTF8_APPEND_CHAR_SAFE(str, offset, size, test[i+1]);
            if(offset != movedOffset[count]){
                log_err("ERROR: UTF8_APPEND_CHAR_SAFE failed to move the offset correctly for count=%d.\nExpectedOffset=%d  currentOffset=%d\n",
                    count, movedOffset[count], offset);

            }
            if(uprv_memcmp(str, result[count], size) !=0){
                log_err("ERROR: UTF8_APPEND_CHAR_SAFE failed for count=%d. \nExpected:", count);
                printUChars(result[count], size);
                log_err("\nGot:     ");
                printUChars(str, size);
                log_err("\n");
            }
            /*call the API instead of MACRO
            uprv_memcpy(str, s, size);
            offset=test[i];
            c=test[i+1];
            if((uint32_t)(c)<=0x7f) {
                  (str)[(offset)++]=(uint8_t)(c);
            } else {
                 (offset)=utf8_appendCharSafeBody(str, (int32_t)(offset), (int32_t)(size), c);
            }
            if(offset != movedOffset[count]){
                log_err("ERROR: utf8_appendCharSafeBody() failed to move the offset correctly for count=%d.\nExpectedOffset=%d  currentOffset=%d\n",
                    count, movedOffset[count], offset);

            }
            if(uprv_memcmp(str, result[count], size) !=0){
                log_err("ERROR: utf8_appendCharSafeBody() failed for count=%d. \nExpected:", count);
                printUChars(result[count], size);
                printf("\nGot:     ");
                printUChars(str, size);
                printf("\n");
            }
            */
        }
        count++;
    }


}

static void TestAppend() {
    static const UChar32 codePoints[]={
        0x61, 0xdf, 0x901, 0x3040,
        0xac00, 0xd800, 0xdbff, 0xdcde,
        0xdffd, 0xe000, 0xffff, 0x10000,
        0x12345, 0xe0021, 0x10ffff, 0x110000,
        0x234567, 0x7fffffff, -1, -1000,
        0, 0x400
    };
    static const uint8_t expectUnsafe[]={
        0x61,  0xc3, 0x9f,  0xe0, 0xa4, 0x81,  0xe3, 0x81, 0x80,
        0xea, 0xb0, 0x80,  0xed, 0xa0, 0x80,  0xed, 0xaf, 0xbf,  0xed, 0xb3, 0x9e,
        0xed, 0xbf, 0xbd,  0xee, 0x80, 0x80,  0xef, 0xbf, 0xbf,  0xf0, 0x90, 0x80, 0x80,
        0xf0, 0x92, 0x8d, 0x85,  0xf3, 0xa0, 0x80, 0xa1,  0xf4, 0x8f, 0xbf, 0xbf,  /* not 0x110000 */
        /* none from this line */
        0,  0xd0, 0x80
    }, expectSafe[]={
        0x61,  0xc3, 0x9f,  0xe0, 0xa4, 0x81,  0xe3, 0x81, 0x80,
        0xea, 0xb0, 0x80,  /* no surrogates */
        /* no surrogates */  0xee, 0x80, 0x80,  0xef, 0xbf, 0xbf,  0xf0, 0x90, 0x80, 0x80,
        0xf0, 0x92, 0x8d, 0x85,  0xf3, 0xa0, 0x80, 0xa1,  0xf4, 0x8f, 0xbf, 0xbf,  /* not 0x110000 */
        /* none from this line */
        0,  0xd0, 0x80
    };

    uint8_t buffer[100];
    UChar32 c;
    int32_t i, length;
    UBool isError, expectIsError, wrongIsError;

    length=0;
    for(i=0; i<LENGTHOF(codePoints); ++i) {
        c=codePoints[i];
        if(c<0 || 0x10ffff<c) {
            continue; /* skip non-code points for U8_APPEND_UNSAFE */
        }

        U8_APPEND_UNSAFE(buffer, length, c);
    }
    if(length!=LENGTHOF(expectUnsafe) || 0!=memcmp(buffer, expectUnsafe, length)) {
        log_err("U8_APPEND_UNSAFE did not generate the expected output\n");
    }

    length=0;
    wrongIsError=FALSE;
    for(i=0; i<LENGTHOF(codePoints); ++i) {
        c=codePoints[i];
        expectIsError= c<0 || 0x10ffff<c || U_IS_SURROGATE(c);
        isError=FALSE;

        U8_APPEND(buffer, length, LENGTHOF(buffer), c, isError);
        wrongIsError|= isError!=expectIsError;
    }
    if(wrongIsError) {
        log_err("U8_APPEND did not set isError correctly\n");
    }
    if(length!=LENGTHOF(expectSafe) || 0!=memcmp(buffer, expectSafe, length)) {
        log_err("U8_APPEND did not generate the expected output\n");
    }
}

static void
TestSurrogates() {
    static const uint8_t b[]={
        0xc3, 0x9f,             /*  00DF */
        0xed, 0x9f, 0xbf,       /*  D7FF */
        0xed, 0xa0, 0x81,       /*  D801 */
        0xed, 0xbf, 0xbe,       /*  DFFE */
        0xee, 0x80, 0x80,       /*  E000 */
        0xf0, 0x97, 0xbf, 0xbe  /* 17FFE */
    };
    static const UChar32 cp[]={
        0xdf, 0xd7ff, 0xd801, 0xdffe, 0xe000, 0x17ffe
    };

    UChar32 cu, cs, cl;
    int32_t i, j, k, iu, is, il, length;

    k=0; /* index into cp[] */
    length=LENGTHOF(b);
    for(i=0; i<length;) {
        j=i;
        U8_NEXT_UNSAFE(b, j, cu);
        iu=j;

        j=i;
        U8_NEXT(b, j, length, cs);
        is=j;

        j=i;
        L8_NEXT(b, j, length, cl);
        il=j;

        if(cu!=cp[k]) {
            log_err("U8_NEXT_UNSAFE(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cu, (long)cp[k]);
        }

        /* U8_NEXT() returns <0 for surrogate code points */
        if(U_IS_SURROGATE(cu) ? cs>=0 : cs!=cu) {
            log_err("U8_NEXT(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cs, (long)cu);
        }

        /* L8_NEXT() returns surrogate code points like U8_NEXT_UNSAFE() */
        if(cl!=cu) {
            log_err("L8_NEXT(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cl, (long)cu);
        }

        if(is!=iu || il!=iu) {
            log_err("U8_NEXT(b[%ld]) or L8_NEXT(b[%ld]) did not advance the index correctly\n", (long)i, (long)i);
        }

        ++k;    /* next code point */
        i=iu;   /* advance by one UTF-8 sequence */
    }

    while(i>0) {
        --k; /* previous code point */

        j=i;
        U8_PREV_UNSAFE(b, j, cu);
        iu=j;

        j=i;
        U8_PREV(b, 0, j, cs);
        is=j;

        j=i;
        L8_PREV(b, 0, j, cl);
        il=j;

        if(cu!=cp[k]) {
            log_err("U8_PREV_UNSAFE(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cu, (long)cp[k]);
        }

        /* U8_PREV() returns <0 for surrogate code points */
        if(U_IS_SURROGATE(cu) ? cs>=0 : cs!=cu) {
            log_err("U8_PREV(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cs, (long)cu);
        }

        /* L8_PREV() returns surrogate code points like U8_PREV_UNSAFE() */
        if(cl!=cu) {
            log_err("L8_PREV(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cl, (long)cu);
        }

        if(is!=iu || il !=iu) {
            log_err("U8_PREV(b[%ld]) or L8_PREV(b[%ld]) did not advance the index correctly\n", (long)i, (long)i);
        }

        i=iu;   /* go back by one UTF-8 sequence */
    }
}

static void printUChars(const uint8_t *uchars, int16_t len){
    int16_t i=0;
    for(i=0; i<len; i++){
        log_err("0x%02x ", *(uchars+i));
    }
}