genidna.c   [plain text]

/*
*******************************************************************************
*
*   Copyright (C) 2003, International Business Machines
*   Corporation and others.  All Rights Reserved.
*
*******************************************************************************
*   file name:  genidna.c
*   encoding:   US-ASCII
*   tab size:   8 (not used)
*   indentation:4
*
*   created on: 2003-02-06
*   created by: Ram Viswanadha
*
*   This program reads the rfc3454_*.txt files,
*   parses them, and extracts the data for Nameprep conformance.
*   It then preprocesses it and writes a binary file for efficient use
*   in various IDNA conversion processes.
*/

#include <stdio.h>
#include <stdlib.h>
#include "unicode/utypes.h"
#include "unicode/uchar.h"
#include "unicode/putil.h"
#include "cmemory.h"
#include "cstring.h"
#include "unicode/udata.h"
#include "unewdata.h"
#include "uoptions.h"
#include "uparse.h"
#include "unicode/uset.h"
#include "uprops.h"

U_CDECL_BEGIN
#include "genidna.h"
U_CDECL_END

#ifdef WIN32
#   pragma warning(disable: 4100)
#endif

UBool beVerbose=FALSE, haveCopyright=TRUE, printRules = FALSE;

/* prototypes --------------------------------------------------------------- */

static void
parseMappings(const char *filename, UBool withNorm, UBool reportError, UErrorCode *pErrorCode);

static void
parseTable(const char *filename, UBool isUnassigned, UErrorCode *pErrorCode);

static void
parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode);

/*static void 
setLDHValues(UErrorCode* pErrorCode);*/

static void
setLabelSeperators(UErrorCode* pErrorCode);

static void 
printMapping(UChar32 cp,UChar32* mapping, int32_t mappingLength);

static const char* fileNames[] = {
                                    "rfc3454_A_1.txt", /* contains unassigned code points */
                                    "rfc3454_C_X.txt", /* contains code points that are prohibited */
                                    "rfc3454_B_1.txt", /* contains case mappings when normalization is turned off */
                                    "rfc3454_B_2.txt", /* contains case mappings when normalization it turned on */
                                    "NormalizationCorrections.txt",/* normalization corrections  */
                                };
static const char *UNIDATA_DIR = "unidata";
static const char *MISC_DIR    = "misc";

/* -------------------------------------------------------------------------- */

static UOption options[]={
    UOPTION_HELP_H,
    UOPTION_HELP_QUESTION_MARK,
    UOPTION_VERBOSE,
    UOPTION_COPYRIGHT,
    UOPTION_DESTDIR,
    UOPTION_SOURCEDIR,
    { "unicode", NULL, NULL, NULL, 'u', UOPT_REQUIRES_ARG, 0 },
    { "generate-rules", NULL, NULL, NULL, 'g', UOPT_NO_ARG, 0 }
};

extern int
main(int argc, char* argv[]) {
#if !UCONFIG_NO_IDNA
    char* filename = NULL;
#endif
    const char *srcDir=NULL, *destDir=NULL, *suffix=NULL;
    char *basename=NULL;
    char *saveBasename = NULL;
    UErrorCode errorCode=U_ZERO_ERROR;

    U_MAIN_INIT_ARGS(argc, argv);

    /* preset then read command line options */
    options[4].value=u_getDataDirectory();
    options[5].value="";
    options[6].value="3.0.0";
    argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);

    /* error handling, printing usage message */
    if(argc<0) {
        fprintf(stderr,
            "error in command line argument \"%s\"\n",
            argv[-argc]);
    }
    if(argc<0 || options[0].doesOccur || options[1].doesOccur) {
        /*
         * Broken into chucks because the C89 standard says the minimum
         * required supported string length is 509 bytes.
         */
        fprintf(stderr,
            "Usage: %s [-options] [suffix]\n"
            "\n"
            "Read the rfc3454_*.txt files and\n"
            "create a binary file " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE " with the normalization data\n"
            "\n",
            argv[0]);
        fprintf(stderr,
            "Options:\n"
            "\t-h or -? or --help		this usage text\n"
            "\t-v or --verbose			verbose output\n"
            "\t-c or --copyright		include a copyright notice\n");
        fprintf(stderr,
            "\t-d or --destdir			destination directory, followed by the path\n"
            "\t-s or --sourcedir		source directory of ICU data, followed by the path\n"
            "\t-g or --generate-rules   generate IDN rules for testing. Will print out rules to STDOUT\n"
            );
        return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
    }

    /* get the options values */
    beVerbose=options[2].doesOccur;
    haveCopyright=options[3].doesOccur;
    srcDir=options[5].value;
    destDir=options[4].value;
    printRules = options[7].doesOccur;

    if(argc>=2) {
        suffix=argv[1];
    } else {
        suffix=NULL;
    }

#if UCONFIG_NO_IDNA

    fprintf(stderr,
        "genidna writes dummy " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE
        " because UCONFIG_NO_IDNA is set, \n"
        "see icu/source/common/unicode/uconfig.h\n");
    generateData(destDir);

#else

    setUnicodeVersion(options[6].value);
    filename = (char* ) uprv_malloc(uprv_strlen(srcDir) + 300); /* hopefully this should be enough */
    /* prepare the filename beginning with the source dir */
    if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL){
        filename[0] = 0x2E;
        filename[1] = U_FILE_SEP_CHAR;
        uprv_strcpy(filename+2,srcDir);
    }else{
        uprv_strcpy(filename, srcDir);
    }
    basename=filename+uprv_strlen(filename);
    if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) {
        *basename++=U_FILE_SEP_CHAR;
    }
    
    /* initialize */
    init();
    if(printRules){
        printf("// Copyright (C) 2003, International Business Machines\n\n");
        printf("// WARNING: This file is machine generated by %s tool. Please DO NOT edit.\n\n",argv[0]);

        printf("idn_rules{\n");
    }

    /* first copy misc directory */
    saveBasename = basename;
    uprv_strcpy(basename,MISC_DIR);
    basename = basename + uprv_strlen(MISC_DIR);
    *basename++=U_FILE_SEP_CHAR;

    /* process unassigned */
    uprv_strcpy(basename,fileNames[0]);
    parseTable(filename,TRUE, &errorCode);
    if(U_FAILURE(errorCode)) {
        fprintf(stderr, "Could not open file %s for reading \n", filename);
        return errorCode;
    }
    /* process prohibited */
    uprv_strcpy(basename,fileNames[1]);
    parseTable(filename,FALSE,  &errorCode);
    if(U_FAILURE(errorCode)) {
        fprintf(stderr, "Could not open file %s for reading \n", filename);
        return errorCode;
    }

    /*  setLDHValues(&errorCode); */
    setLabelSeperators(&errorCode);

    /* process mappings */
    if(printRules){
        printf("\n\tMapNoNormalization{\n");
    }
    uprv_strcpy(basename,fileNames[2]);
    parseMappings(filename, FALSE, FALSE, &errorCode);
    if(U_FAILURE(errorCode)) {
        fprintf(stderr, "Could not open file %s for reading \n", filename);
        return errorCode;
    }
    if(printRules){
        printf("\n\t}\n");
    }
    
    if(printRules){
        printf("\n\tMapNFKC{\n");
    }
    uprv_strcpy(basename,fileNames[3]);
    parseMappings(filename, TRUE, FALSE, &errorCode);
    if(U_FAILURE(errorCode)) {
        fprintf(stderr, "Could not open file %s for reading \n", filename);
        return errorCode;
    }
    /* set up directory for NormalizationCorrections.txt */
    basename = saveBasename;
    uprv_strcpy(basename,UNIDATA_DIR);
    basename = basename + uprv_strlen(UNIDATA_DIR);
    *basename++=U_FILE_SEP_CHAR;
    uprv_strcpy(basename,fileNames[4]);
    
    parseNormalizationCorrections(filename,&errorCode);
    if(U_FAILURE(errorCode)){
        fprintf(stderr,"Could not open file %s for reading \n", filename);
        return errorCode;
    }

    /* process parsed data */
    if(U_SUCCESS(errorCode)) {
        /* write the data file */
       generateData(destDir);

       cleanUpData();
    }
    if(printRules){
        printf("\t\t\"::[:AGE=3.2:]NFKC;\"\n\t}\n}");
    }

    uprv_free(filename);

#endif

    return errorCode;
}

#if !UCONFIG_NO_IDNA

static void U_CALLCONV
normalizationCorrectionsLineFn(void *context,
                    char *fields[][2], int32_t fieldCount,
                    UErrorCode *pErrorCode) {
    uint32_t mapping[40];
    char *end, *s;
    uint32_t code;
    int32_t length;
    UVersionInfo version;
    UVersionInfo thisVersion;

    /* get the character code, field 0 */
    code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
    if(U_FAILURE(*pErrorCode)) {
        fprintf(stderr, "genidn: error parsing FCNFKC_3_2_0.txt mapping at %s\n", fields[0][0]);
        exit(*pErrorCode);
    }
    /* Original (erroneous) decomposition */
    s = fields[1][0];

    /* parse the mapping string */
    length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);

    /* ignore corrected decomposition */

    u_versionFromString(version,fields[3][0] );
    u_versionFromString(thisVersion, "3.2.0");



    if(U_FAILURE(*pErrorCode)) {
        fprintf(stderr, "genidn error parsing NormalizationCorrection of U+%04lx - %s\n",
                (long)code, u_errorName(*pErrorCode));
        exit(*pErrorCode);
    }

    /* store the mapping */
    if( version[0] > thisVersion[0] || 
        ((version[0]==thisVersion[0]) && (version[1] > thisVersion[1]))
        ){
        storeMapping(code,mapping, length, TRUE, pErrorCode);
        if(printRules){
            printMapping(code,(UChar32*)mapping,length);
        }
    }
}

static void
parseNormalizationCorrections(const char *filename, UErrorCode *pErrorCode) {
    char *fields[4][2];

    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
        return;
    }

    u_parseDelimitedFile(filename, ';', fields, 4, normalizationCorrectionsLineFn, NULL, pErrorCode);

    /* fprintf(stdout,"Number of code points that have NormalizationCorrections mapping with length >1 : %i\n",len); */

    if(U_FAILURE(*pErrorCode) && ( *pErrorCode!=U_FILE_ACCESS_ERROR)) {
        fprintf(stderr, "genidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
        exit(*pErrorCode);
    }
}

static void U_CALLCONV
caseMapLineFn(void *context,
              char *fields[][2], int32_t fieldCount,
              UErrorCode *pErrorCode) {
    uint32_t mapping[40];
    char *end, *s;
    uint32_t code;
    int32_t length;
    UBool* mapWithNorm = (UBool*) context;


    /* get the character code, field 0 */
    code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16);
    if(end<=fields[0][0] || end!=fields[0][1]) {
        fprintf(stderr, "genidn: syntax error in field 0 at %s\n", fields[0][0]);
        *pErrorCode=U_PARSE_ERROR;
        exit(U_PARSE_ERROR);
    }

    s = fields[1][0];
    /* parse the mapping string */
    length=u_parseCodePoints(s, mapping, sizeof(mapping)/4, pErrorCode);

    if(U_FAILURE(*pErrorCode)) {
        fprintf(stderr, "genidn error parsing UnicodeData.txt decomposition of U+%04lx - %s\n",
                (long)code, u_errorName(*pErrorCode));
        exit(*pErrorCode);
    }

    /* store the mapping */

    storeMapping(code,mapping, length, *mapWithNorm, pErrorCode);
    if(printRules){
        printMapping(code,(UChar32*)mapping,length);
    }
}

static void
parseMappings(const char *filename,UBool withNorm, UBool reportError, UErrorCode *pErrorCode) {
    char *fields[3][2];

    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
        return;
    }

    u_parseDelimitedFile(filename, ';', fields, 3, caseMapLineFn, &withNorm, pErrorCode);

    /*fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len);*/

    if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) {
        fprintf(stderr, "genidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
        exit(*pErrorCode);
    }
}

/* parser for UnicodeData.txt ----------------------------------------------- */
static int32_t printedCharCount = 0;

static void printEscaped(UChar32 ch){
    if(ch > 0xFFFF){
        printf("\\\\U%08X",ch);
        printedCharCount+=11;
    }else{
        if(uprv_isRuleWhiteSpace(ch)){
            /* double escape the rule white space */
            printf("\\\\u%04X", ch);
            printedCharCount+=7;
        }else if(0x20< ch && ch <0x7f){
            if(ch == 0x2E){
                /* double escape dot */
                printf("\\\\%c",(char)ch);
                printedCharCount+=3;
            }else{
                printf("%c",(char)ch);
                printedCharCount++;
            }
        }else{
            printf("\\\\u%04X",ch);
            printedCharCount+=7;
        }
    }
}
static void printEscapedRange(UChar32 rangeStart, UChar32 rangeEnd){
    if(rangeStart != rangeEnd){
        printEscaped(rangeStart);
        printf("-");
        printedCharCount++;
        printEscaped(rangeEnd);
        printf(" ");
    }else{
        printEscaped(rangeStart);
        printf(" ");
    }
    if(printedCharCount > 70){
        printf("\"\n\t\t\t\"");
        printedCharCount =0 ;
    }
}
static void printMapping( UChar32 cp, UChar32* mapping, int32_t mappingLength){
    
    int32_t i;
    printf("\t\t\"");
    printEscaped(cp);
    printf(" > ");
    for(i=0;i<mappingLength;i++){
        printEscaped(mapping[i]);
    }
    printf(";\"\n");
    
    printedCharCount=0; 
}
static void U_CALLCONV
unicodeDataLineFn(void *context,
                  char *fields[][2], int32_t fieldCount,
                  UErrorCode *pErrorCode) {
    uint32_t rangeStart=0,rangeEnd =0;
    UBool* isUnassigned = (UBool*) context;


    u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode);
    
    if(U_FAILURE(*pErrorCode)){
        fprintf(stderr, "Could not parse code point range. Error: %s\n",u_errorName(*pErrorCode));
        return;
    }

    if(*isUnassigned == TRUE){
        storeRange(rangeStart,rangeEnd,UIDNA_UNASSIGNED, pErrorCode);
    }else{
        storeRange(rangeStart,rangeEnd,UIDNA_PROHIBITED, pErrorCode);
    }
    /*TODO: comment out the printer */
    if(printRules){
        printEscapedRange(rangeStart,rangeEnd);
    }
}

static void
parseTable(const char *filename,UBool isUnassigned, UErrorCode *pErrorCode) {
    char *fields[1][2];
    if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) {
        return;
    }
    /*TODO: comment out the printer */
    if(printRules){
        printedCharCount = 0;
        if(isUnassigned){
            printf("\n\tUnassignedSet{\"[ ");
        }else{
            printf("\n\tProhibitedSet{\"[ ");
        }
    }
    u_parseDelimitedFile(filename, ';', fields, 1, unicodeDataLineFn, &isUnassigned, pErrorCode);


    if(U_FAILURE(*pErrorCode)) {
        fprintf(stderr, "genidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode));
        exit(*pErrorCode);
    }
    if(printRules){
        printf("]\"}\n");
    }
}

/*
static void 
setLDHValues(UErrorCode* pErrorCode){
    USet* set = uset_openPattern(LDH_PATTERN, LDH_PATTERN_LEN, pErrorCode);
    int32_t itemCount;
    int32_t index = 0;
    UChar32 start,end;

    if(U_FAILURE(*pErrorCode)){
        fprintf(stderr,"Could not open USet. Error :%s \n",u_errorName(*pErrorCode));
        exit(*pErrorCode);
    }
    
    itemCount = uset_getItemCount(set);

    for(;index < itemCount; index++){
        uset_getItem(set,index, &start, &end, NULL, 0, pErrorCode);
        storeRange(start,end,UIDNA_LDH_OR_MAP_NFKC, pErrorCode);
    }
    if(printRules){
        printf(PAT);
    }

}
*/
static void
setLabelSeperators(UErrorCode *pErrorCode){
    /* U+002E, U+3002, U+FF0E, U+FF61 */
    storeRange(0x002E, 0x002E, UIDNA_LABEL_SEPARATOR, pErrorCode);
    storeRange(0x3002, 0x3002, UIDNA_LABEL_SEPARATOR, pErrorCode);
    storeRange(0xFF0E, 0xFF0E, UIDNA_LABEL_SEPARATOR, pErrorCode);
    storeRange(0xFF61, 0xFF61, UIDNA_LABEL_SEPARATOR, pErrorCode);
    if(U_FAILURE(*pErrorCode)){
        fprintf(stderr, "Could not store values for label separators\n");
    }
    if(printRules){
        printf("\tLabelSeparatorSet{\"[ ");
        printEscaped(0x002E);
        printEscaped(0x3002);
        printEscaped(0xFF0E);
        printEscaped(0xFF61);
        printf(" ]\"}\n\n");
    }
}

#endif /* #if !UCONFIG_NO_IDNA */

/*
 * Hey, Emacs, please set the following:
 *
 * Local Variables:
 * indent-tabs-mode: nil
 * End:
 *
 */