generateYarrUnicodePropertyTables.py [plain text]
import sys
import copy
import optparse
import os
import re
from hasher import stringHash
header = """/*
* Copyright (C) 2017-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
// DO NO EDIT! - This file was generated by """ + __file__ + """
"""
footer = """
"""
RequiredUCDFiles = ["DerivedBinaryProperties.txt", "DerivedCoreProperties.txt", "DerivedNormalizationProps.txt", "PropList.txt", "PropertyAliases.txt", "PropertyValueAliases.txt", "ScriptExtensions.txt", "UnicodeData.txt", "emoji-data.txt"]
UCDDirectoryPath = None
SupportedBinaryProperties = [
"Alphabetic", "Any", "ASCII", "ASCII_Hex_Digit", "Assigned", "Bidi_Control", "Bidi_Mirrored", "Case_Ignorable",
"Cased", "Changes_When_Casefolded", "Changes_When_Casemapped", "Changes_When_Lowercased", "Changes_When_NFKC_Casefolded",
"Changes_When_Titlecased", "Changes_When_Uppercased", "Dash", "Default_Ignorable_Code_Point", "Deprecated",
"Diacritic", "Emoji", "Emoji_Component", "Emoji_Modifier_Base", "Emoji_Modifier", "Emoji_Presentation",
"Extended_Pictographic", "Extender", "Grapheme_Base", "Grapheme_Extend", "Hex_Digit", "ID_Continue", "ID_Start",
"Ideographic", "IDS_Binary_Operator", "IDS_Trinary_Operator", "Join_Control", "Logical_Order_Exception", "Lowercase",
"Math", "Noncharacter_Code_Point", "Pattern_Syntax", "Pattern_White_Space", "Quotation_Mark", "Radical",
"Regional_Indicator", "Sentence_Terminal", "Soft_Dotted", "Terminal_Punctuation", "Unified_Ideograph", "Uppercase",
"Variation_Selector", "White_Space", "XID_Continue", "XID_Start"]
lastASCIICodePoint = 0x7f
firstUnicodeCodePoint = 0x80
MaxUnicode = 0x10ffff
MaxBMP = 0xffff
commonAndSimpleLinesRE = re.compile(r"(?P<code>[0-9A-F]+)\s*;\s*[CS]\s*;\s*(?P<mapping>[0-9A-F]+)", re.IGNORECASE)
aliases = None
def openOrExit(path, mode):
try:
return open(path, mode)
except IOError as e:
print("I/O error opening {0}, ({1}): {2}".format(path, e.errno, e.strerror))
exit(1)
def openUCDFileOrExit(path):
if not UCDDirectoryPath:
exit(1)
return openOrExit(os.path.join(UCDDirectoryPath, path), 'r')
def verifyUCDFilesExist():
if not UCDDirectoryPath:
exit(1)
missingFileCount = 0
for file in RequiredUCDFiles:
fullPath = os.path.join(UCDDirectoryPath, file)
if not os.path.exists(fullPath):
print("Couldn't find UCD file {0} at {1}".format(file, fullPath))
missingFileCount = missingFileCount + 1
if missingFileCount:
exit(1)
def ceilingToPowerOf2(size):
powerOf2 = 1
while size > powerOf2:
powerOf2 = powerOf2 << 1
return powerOf2
class Aliases:
def __init__(self):
self.globalNameToAliases = {}
self.generalCategoryToAliases = {}
self.aliasToGeneralCategory = {}
self.scriptToAliases = {}
self.aliasToScript = {}
def parsePropertyAliasesFile(self, file):
for line in file:
line = line.split('#', 1)[0]
line = line.rstrip()
if (not len(line)):
continue
fields = line.split(';')
if (not fields):
continue
aliases = [fields[0].strip()]
fullName = fields[1].strip()
for otherAlias in fields[2:]:
aliases.append(otherAlias.strip())
if fullName in self.globalNameToAliases:
print("Error, already an alias for {}".format(fullName))
else:
self.globalNameToAliases[fullName] = aliases
def parsePropertyValueAliasesFile(self, file):
for line in file:
line = line.split('#', 1)[0]
line = line.rstrip()
if (not len(line)):
continue
fields = line.split(';')
if (not fields):
continue
propertyType = fields[0].strip()
if propertyType == "gc":
mapToModify = self.generalCategoryToAliases
reverseMapToModify = self.aliasToGeneralCategory
elif propertyType == "sc":
mapToModify = self.scriptToAliases
reverseMapToModify = self.aliasToScript
else:
continue
primaryAlias = fields[1].strip()
fullName = fields[2].strip()
aliases = [primaryAlias]
for otherAlias in fields[3:]:
aliases.append(otherAlias.strip())
if fullName in mapToModify:
print("Error, already an {} alias for {}".format(propertyType, fullName))
else:
mapToModify[fullName] = aliases
if reverseMapToModify != None:
reverseMapToModify[primaryAlias] = fullName
def globalAliasesFor(self, name):
if name not in self.globalNameToAliases:
return []
return self.globalNameToAliases[name]
def generalCategoryAliasesFor(self, name):
if name not in self.generalCategoryToAliases:
return ""
return self.generalCategoryToAliases[name]
def generalCategoryForAlias(self, name):
if name not in self.aliasToGeneralCategory:
return ""
return self.aliasToGeneralCategory[name]
def scriptAliasesFor(self, name):
if name not in self.scriptToAliases:
return ""
return self.scriptToAliases[name]
def scriptNameForAlias(self, name):
if name not in self.aliasToScript:
return ""
return self.aliasToScript[name]
class PropertyData:
allPropertyData = []
def __init__(self, name):
self.name = name
self.aliases = []
self.index = len(PropertyData.allPropertyData)
self.hasNonBMPCharacters = False
self.matches = []
self.ranges = []
self.unicodeMatches = []
self.unicodeRanges = []
self.codePointCount = 0
PropertyData.allPropertyData.append(self)
def setAliases(self, aliases):
self.aliases = aliases
def makeCopy(self):
result = copy.deepcopy(self)
result.index = len(PropertyData.allPropertyData)
PropertyData.allPropertyData.append(result)
return result
def getIndex(self):
return self.index
def getCreateFuncName(self):
return "createCharacterClass{}".format(self.index)
def addMatch(self, codePoint):
if codePoint > MaxBMP:
self.hasNonBMPCharacters = True
if codePoint <= lastASCIICodePoint:
if (len(self.matches) and self.matches[-1] > codePoint) or (len(self.ranges) and self.ranges[-1][1] > codePoint):
self.addMatchUnordered(codePoint)
return
self.codePointCount = self.codePointCount + 1
if len(self.matches) and self.matches[-1] == (codePoint - 1):
lowCodePoint = self.matches.pop()
self.ranges.append((lowCodePoint, codePoint))
elif len(self.ranges) and self.ranges[-1][1] == (codePoint - 1):
priorRange = self.ranges.pop()
self.ranges.append((priorRange[0], codePoint))
else:
self.matches.append(codePoint)
else:
if (len(self.unicodeMatches) and self.unicodeMatches[-1] > codePoint) or (len(self.unicodeRanges) and self.unicodeRanges[-1][1] > codePoint):
self.addMatchUnordered(codePoint)
return
self.codePointCount = self.codePointCount + 1
if len(self.unicodeMatches) and self.unicodeMatches[-1] == (codePoint - 1):
lowCodePoint = self.unicodeMatches.pop()
self.unicodeRanges.append((lowCodePoint, codePoint))
elif len(self.unicodeRanges) and self.unicodeRanges[-1][1] == (codePoint - 1):
priorRange = self.unicodeRanges.pop()
self.unicodeRanges.append((priorRange[0], codePoint))
else:
self.unicodeMatches.append(codePoint)
def addRange(self, lowCodePoint, highCodePoint):
if highCodePoint > MaxBMP:
self.hasNonBMPCharacters = True
if highCodePoint <= lastASCIICodePoint:
if (len(self.matches) and self.matches[-1] > lowCodePoint) or (len(self.ranges) and self.ranges[-1][1] > lowCodePoint):
self.addRangeUnordered(lowCodePoint, highCodePoint)
return
self.codePointCount = self.codePointCount + (highCodePoint - lowCodePoint) + 1
if len(self.matches) and self.matches[-1] == (lowCodePoint - 1):
lowCodePoint = self.matches.pop()
elif len(self.ranges) and self.ranges[-1][1] == (lowCodePoint - 1):
priorRange = self.ranges.pop()
lowCodePoint = priorRange[0]
self.ranges.append((lowCodePoint, highCodePoint))
elif lowCodePoint <= lastASCIICodePoint:
if lowCodePoint == lastASCIICodePoint:
self.addMatch(lowCodePoint)
else:
self.addRange(lowCodePoint, lastASCIICodePoint)
if highCodePoint == firstUnicodeCodePoint:
self.addMatch(highCodePoint)
else:
self.addRange(firstUnicodeCodePoint, highCodePoint)
else:
if (len(self.unicodeMatches) and self.unicodeMatches[-1] > lowCodePoint) or (len(self.unicodeRanges) and self.unicodeRanges[-1][1] > lowCodePoint):
self.addRangeUnordered(lowCodePoint, highCodePoint)
return
self.codePointCount = self.codePointCount + (highCodePoint - lowCodePoint) + 1
if len(self.unicodeMatches) and self.unicodeMatches[-1] == (lowCodePoint - 1):
lowCodePoint = self.unicodeMatches.pop()
self.unicodeRanges.append((lowCodePoint, highCodePoint))
elif len(self.unicodeRanges) and self.unicodeRanges[-1][1] == (lowCodePoint - 1):
priorRange = self.unicodeRanges.pop()
self.unicodeRanges.append((priorRange[0], highCodePoint))
else:
self.unicodeRanges.append((lowCodePoint, highCodePoint))
def addMatchUnorderedForMatchesAndRanges(self, codePoint, matches, ranges):
if codePoint in matches:
return
insertLocation = None
lowCodePoint = None
highCodePoint = None
for idx in xrange(len(matches)):
match = matches[idx]
if codePoint == match + 1:
lowCodePoint = match
if idx < (len(matches) - 1) and codePoint == matches[idx + 1] - 1:
highCodePoint = matches[idx + 1]
del matches[idx + 1]
self.codePointCount = self.codePointCount - 1
else:
highCodePoint = codePoint
del matches[idx]
self.codePointCount = self.codePointCount - 1
break
elif codePoint == match - 1:
lowCodePoint = codePoint
highCodePoint = match
del matches[idx]
self.codePointCount = self.codePointCount - 1
break
elif codePoint < match:
insertLocation = idx
break
if insertLocation is None:
insertLocation = len(matches)
if lowCodePoint is None:
lowCodePoint = codePoint
highCodePoint = codePoint
for idx in xrange(len(ranges)):
range = ranges[idx]
if lowCodePoint >= range[0] and highCodePoint <= range[1]:
return
if lowCodePoint <= (range[1] + 1) and highCodePoint >= (range[0] - 1):
while idx < len(ranges) and highCodePoint >= (ranges[idx][0] - 1):
range = ranges[idx]
lowCodePoint = min(lowCodePoint, range[0])
highCodePoint = max(highCodePoint, range[1])
del ranges[idx]
self.codePointCount = self.codePointCount - (range[1] - range[0]) - 1
ranges.insert(idx, (lowCodePoint, highCodePoint))
self.codePointCount = self.codePointCount + (highCodePoint - lowCodePoint) + 1
return
elif highCodePoint < range[0]:
if lowCodePoint != highCodePoint:
ranges.insert(idx, (lowCodePoint, highCodePoint))
self.codePointCount = self.codePointCount + (highCodePoint - lowCodePoint) + 1
return
break
if lowCodePoint != highCodePoint:
ranges.append((lowCodePoint, highCodePoint))
self.codePointCount = self.codePointCount + (highCodePoint - lowCodePoint) + 1
return
if insertLocation is not None:
matches.insert(insertLocation, codePoint)
self.codePointCount = self.codePointCount + 1
def addRangeUnorderedForMatchesAndRanges(self, lowCodePoint, highCodePoint, matches, ranges):
if len(matches) and highCodePoint >= matches[0] and lowCodePoint <= matches[-1]:
for idx in xrange(len(matches)):
match = matches[idx]
if lowCodePoint <= match and highCodePoint >= match:
while idx < len(matches) and highCodePoint >= matches[idx]:
del matches[idx]
self.codePointCount = self.codePointCount - 1
if idx < (len(matches) - 1) and highCodePoint == matches[idx + 1] - 1:
highCodePoint = matches[idx + 1]
del matches[idx + 1]
self.codePointCount = self.codePointCount - 1
break
elif lowCodePoint == match + 1:
lowCodePoint = match
while idx < len(matches) and highCodePoint >= matches[idx]:
del matches[idx]
self.codePointCount = self.codePointCount - 1
if idx < (len(matches) - 1) and highCodePoint == matches[idx + 1] - 1:
highCodePoint = matches[idx + 1]
del matches[idx + 1]
self.codePointCount = self.codePointCount - 1
break
elif highCodePoint == match - 1:
highCodePoint = match
del matches[idx]
self.codePointCount = self.codePointCount - 1
break
elif highCodePoint < match:
break
for idx in xrange(len(ranges)):
range = ranges[idx]
if lowCodePoint >= range[0] and highCodePoint <= range[1]:
return
if lowCodePoint <= (range[1] + 1) and highCodePoint >= (range[0] - 1):
while idx < len(ranges) and highCodePoint >= (ranges[idx][0] - 1):
range = ranges[idx]
lowCodePoint = min(lowCodePoint, range[0])
highCodePoint = max(highCodePoint, range[1])
del ranges[idx]
self.codePointCount = self.codePointCount - (range[1] - range[0]) - 1
ranges.insert(idx, (lowCodePoint, highCodePoint))
self.codePointCount = self.codePointCount + (highCodePoint - lowCodePoint) + 1
return
elif highCodePoint < range[0]:
ranges.insert(idx, (lowCodePoint, highCodePoint))
self.codePointCount = self.codePointCount + (highCodePoint - lowCodePoint) + 1
return
ranges.append((lowCodePoint, highCodePoint))
self.codePointCount = self.codePointCount + (highCodePoint - lowCodePoint) + 1
def addMatchUnordered(self, codePoint):
if codePoint <= lastASCIICodePoint:
self.addMatchUnorderedForMatchesAndRanges(codePoint, self.matches, self.ranges)
else:
self.addMatchUnorderedForMatchesAndRanges(codePoint, self.unicodeMatches, self.unicodeRanges)
def addRangeUnordered(self, lowCodePoint, highCodePoint):
if highCodePoint <= lastASCIICodePoint:
self.addRangeUnorderedForMatchesAndRanges(lowCodePoint, highCodePoint, self.matches, self.ranges)
elif lowCodePoint >= firstUnicodeCodePoint:
self.addRangeUnorderedForMatchesAndRanges(lowCodePoint, highCodePoint, self.unicodeMatches, self.unicodeRanges)
else:
if lowCodePoint == lastASCIICodePoint:
self.addMatchUnorderedForMatchesAndRanges(lowCodePoint, self.matches, self.ranges)
else:
self.addRangeUnorderedForMatchesAndRanges(lowCodePoint, lastASCIICodePoint, self.unicodeMatches, self.ranges)
if highCodePoint == firstUnicodeCodePoint:
self.addMatchUnorderedForMatchesAndRanges(highCodePoint, self.unicodeMatches, self.unicodeRanges)
else:
self.addRangeUnorderedForMatchesAndRanges(firstUnicodeCodePoint, highCodePoint, self.unicodeMatches, self.unicodeRanges)
def removeMatchFromRanges(self, codePoint, ranges):
for idx in xrange(len(ranges)):
range = ranges[idx]
if range[0] <= codePoint and codePoint <= range[1]:
ranges.pop(idx)
if range[0] < codePoint and codePoint < range[1]:
lowRange = (range[0], codePoint - 1)
highRange = (codePoint + 1, range[1])
if highRange[0] == highRange[1]:
self.addMatchUnordered(highRange[0])
else:
ranges.insert(idx, highRange)
if lowRange[0] == lowRange[1]:
self.addMatchUnordered(lowRange[0])
else:
ranges.insert(idx, lowRange)
else:
if range[0] == codePoint:
range = (codePoint + 1, range[1])
else:
range = (range[0], codePoint - 1)
if range[0] == range[1]:
self.addMatchUnordered(range[0])
else:
ranges.insert(idx, range)
self.codePointCount = self.codePointCount - 1
return
def removeMatch(self, codePoint):
if codePoint <= lastASCIICodePoint:
if codePoint in self.matches:
self.matches.remove(codePoint)
self.codePointCount = self.codePointCount - 1
else:
self.removeMatchFromRanges(codePoint, self.ranges)
else:
if codePoint in self.unicodeMatches:
self.unicodeMatches.remove(codePoint)
self.codePointCount = self.codePointCount - 1
else:
self.removeMatchFromRanges(codePoint, self.unicodeRanges)
def dumpMatchData(self, file, valuesPerLine, dataList, formatter):
valuesThisLine = 0
firstValue = True
file.write("{")
for elem in dataList:
if firstValue:
firstValue = False
else:
file.write(", ")
valuesThisLine = valuesThisLine + 1
if valuesThisLine > valuesPerLine:
file.write("\n ")
valuesThisLine = 1
formatter(file, elem)
file.write("}")
def dump(self, file, commaAfter):
file.write("static std::unique_ptr<CharacterClass> {}()\n{{\n".format(self.getCreateFuncName()))
file.write(" // Name = {}, number of codePoints: {}\n".format(self.name, self.codePointCount))
file.write(" auto characterClass = std::make_unique<CharacterClass>(\n")
file.write(" std::initializer_list<UChar32>(")
self.dumpMatchData(file, 8, self.matches, lambda file, match: (file.write("{0:0=#4x}".format(match))))
file.write("),\n")
file.write(" std::initializer_list<CharacterRange>(")
self.dumpMatchData(file, 4, self.ranges, lambda file, range: (file.write("{{{0:0=#4x}, {1:0=#4x}}}".format(range[0], range[1]))))
file.write("),\n")
file.write(" std::initializer_list<UChar32>(")
self.dumpMatchData(file, 8, self.unicodeMatches, lambda file, match: (file.write("{0:0=#6x}".format(match))))
file.write("),\n")
file.write(" std::initializer_list<CharacterRange>(")
self.dumpMatchData(file, 4, self.unicodeRanges, lambda file, range: (file.write("{{{0:0=#6x}, {1:0=#6x}}}".format(range[0], range[1]))))
file.write("));\n")
file.write(" characterClass->m_hasNonBMPCharacters = {};\n".format(("false", "true")[self.hasNonBMPCharacters]))
file.write(" return characterClass;\n}\n\n")
@classmethod
def dumpAll(cls, file):
for propertyData in cls.allPropertyData:
propertyData.dump(file, propertyData != cls.allPropertyData[-1])
file.write("typedef std::unique_ptr<CharacterClass> (*CreateCharacterClass)();\n")
file.write("static CreateCharacterClass createFunctions[{}] = {{\n ".format(len(cls.allPropertyData)))
functionsOnThisLine = 0
for propertyData in cls.allPropertyData:
file.write(" {},".format(propertyData.getCreateFuncName()))
functionsOnThisLine = functionsOnThisLine + 1
if functionsOnThisLine == 4:
file.write("\n ")
functionsOnThisLine = 0
file.write("};\n\n")
@classmethod
def createAndDumpHashTable(self, file, propertyDict, tablePrefix):
propertyKeys = propertyDict.keys()
numberOfKeys = len(propertyKeys)
hashSize = ceilingToPowerOf2(numberOfKeys * 2)
hashMask = hashSize - 1
hashTable = [None] * hashSize
valueTable = []
tableSize = hashSize
keyValuesToHash = []
for propertyName in propertyKeys:
propertyData = propertyDict[propertyName]
keyValuesToHash.append((propertyName, propertyData.getIndex()))
for alias in propertyData.aliases:
keyValuesToHash.append((alias, propertyData.getIndex()))
for keyValue in keyValuesToHash:
key = keyValue[0]
hash = stringHash(key) % hashSize
while hashTable[hash] is not None:
if hashTable[hash][1] is not None:
hash = hashTable[hash][1]
else:
hashTable[hash] = (hashTable[hash][0], tableSize)
hashTable.append(None)
hash = tableSize
tableSize = tableSize + 1
hashTable[hash] = (len(valueTable), None)
valueTable.append((key, keyValue[1]))
file.write("static const struct HashIndex {}TableIndex[{}] = {{\n".format(tablePrefix, len(hashTable)))
for tableIndex in hashTable:
value = -1
next = -1
if tableIndex is not None:
value = tableIndex[0]
if tableIndex[1] is not None:
next = tableIndex[1]
file.write(" {{ {}, {} }},\n".format(value, next))
file.write("};\n\n")
file.write("static const struct HashValue {}TableValue[{}] = {{\n".format(tablePrefix, len(valueTable)))
for value in valueTable:
file.write(" {{ \"{}\", {} }},\n".format(value[0], value[1]))
file.write("};\n\n")
file.write("static const struct HashTable {}HashTable = \n".format(tablePrefix))
file.write(" {{ {}, {}, {}TableValue, {}TableIndex }};\n\n".format(len(valueTable), hashMask, tablePrefix, tablePrefix))
class Scripts:
def __init__(self):
self.allPropertyData = []
self.scriptsByName = {}
self.scriptExtensionsByName = {}
self.unknownScript = PropertyData("Unknown")
self.unknownScript.setAliases(aliases.scriptAliasesFor("Unknown"))
self.allPropertyData.append(self.unknownScript)
self.scriptsParsed = False
def parseScriptsFile(self, file):
currentScriptName = None
currentPropertyData = None
assignedCodePointRanges = []
for line in file:
line = line.split('#', 1)[0]
line = line.rstrip()
if (not len(line)):
continue
fields = line.split(';')
if (not fields):
continue
codePoints = fields[0].strip()
scriptName = fields[1].strip()
if scriptName != currentScriptName:
currentScriptName = scriptName
currentPropertyData = PropertyData(scriptName)
currentPropertyData.setAliases(aliases.scriptAliasesFor(scriptName))
self.allPropertyData.append(currentPropertyData)
self.scriptsByName[scriptName] = currentPropertyData
dotDot = codePoints.find("..")
if dotDot == -1:
codePoint = int(codePoints, 16)
currentPropertyData.addMatch(codePoint)
assignedCodePointRanges.append((codePoint, codePoint))
else:
lowCodePoint = int(codePoints[:dotDot], 16)
highCodePoint = int(codePoints[dotDot + 2:], 16)
currentPropertyData.addRange(lowCodePoint, highCodePoint)
assignedCodePointRanges.append((lowCodePoint, highCodePoint))
assignedCodePointRanges.sort(key=lambda range: range[0])
lastAssignedCodePoint = 0
for range in assignedCodePointRanges:
if range[0] - lastAssignedCodePoint > 1:
if range[0] - lastAssignedCodePoint == 2:
self.unknownScript.addMatch(lastAssignedCodePoint + 1)
else:
self.unknownScript.addRange(lastAssignedCodePoint + 1, range[0] - 1)
lastAssignedCodePoint = range[1]
if lastAssignedCodePoint < MaxUnicode:
if MaxUnicode - lastAssignedCodePoint == 1:
self.unknownScript.addMatch(MaxUnicode)
else:
self.unknownScript.addRange(lastAssignedCodePoint + 1, MaxUnicode)
self.scriptsParsed = True
def parseScriptExtensionsFile(self, file):
currentPropertyData = None
assignedCodePointRanges = []
if not self.scriptsParsed:
print("Error: parsing ScriptExtensions.txt before Scripts.txt")
exit(1)
commonScriptExtenstionPropertyData = None
inheritedScriptExtensionPropertyData = None
scriptName = "Common"
if scriptName in self.scriptsByName:
commonScriptExtenstionPropertyData = self.scriptsByName[scriptName].makeCopy()
else:
commonScriptExtenstionPropertyData = PropertyData(scriptName)
commonScriptExtenstionPropertyData.setAliases(aliases.scriptAliasesFor(scriptName))
self.allPropertyData.append(commonScriptExtenstionPropertyData)
self.scriptExtensionsByName[scriptName] = commonScriptExtenstionPropertyData
scriptName = "Inherited"
if scriptName in self.scriptsByName:
inheritedScriptExtensionPropertyData = self.scriptsByName[scriptName].makeCopy()
else:
inheritedScriptExtensionPropertyData = PropertyData(scriptName)
inheritedScriptExtensionPropertyData.setAliases(aliases.scriptAliasesFor(scriptName))
self.allPropertyData.append(inheritedScriptExtensionPropertyData)
self.scriptExtensionsByName[scriptName] = inheritedScriptExtensionPropertyData
for line in file:
line = line.split('#', 1)[0]
line = line.rstrip()
if (not len(line)):
continue
fields = line.split(';')
if (not fields):
continue
codePoints = fields[0].strip()
scriptAliasList = fields[1].strip()
for scriptAlias in scriptAliasList.split(' '):
scriptName = aliases.scriptNameForAlias(scriptAlias)
currentPropertyData = None
if scriptName not in self.scriptExtensionsByName:
currentPropertyData = self.scriptsByName[scriptName].makeCopy()
self.allPropertyData.append(currentPropertyData)
self.scriptExtensionsByName[scriptName] = currentPropertyData
else:
currentPropertyData = self.scriptExtensionsByName[scriptName]
dotDot = codePoints.find("..")
if dotDot == -1:
codePoint = int(codePoints, 16)
currentPropertyData.addMatch(codePoint)
commonScriptExtenstionPropertyData.removeMatch(codePoint)
inheritedScriptExtensionPropertyData.removeMatch(codePoint)
else:
lowCodePoint = int(codePoints[:dotDot], 16)
highCodePoint = int(codePoints[dotDot + 2:], 16)
currentPropertyData.addRange(lowCodePoint, highCodePoint)
for codePoint in xrange(lowCodePoint, highCodePoint + 1):
commonScriptExtenstionPropertyData.removeMatch(codePoint)
inheritedScriptExtensionPropertyData.removeMatch(codePoint)
for scriptName, propertyData in self.scriptsByName.iteritems():
if scriptName not in self.scriptExtensionsByName:
self.scriptExtensionsByName[scriptName] = propertyData
def dump(self, file):
file.write("// Scripts:\n")
PropertyData.createAndDumpHashTable(file, self.scriptsByName, "script")
file.write("// Script_Extensions:\n")
PropertyData.createAndDumpHashTable(file, self.scriptExtensionsByName, "scriptExtension")
class GeneralCategory:
def __init__(self, file):
self.file = file
self.allPropertyData = []
self.propertyDataByCategory = {}
self.createSpecialPropertyData("Any", (0, MaxUnicode))
self.createSpecialPropertyData("ASCII", (0, lastASCIICodePoint))
self.assignedPropertyData = self.createSpecialPropertyData("Assigned")
self.unassignedProperyData = self.findPropertyGroupFor("Cn")[1]
self.casedLetterPropertyData = self.findPropertyGroupFor("LC")[1]
self.lastAddedCodePoint = 0
def createSpecialPropertyData(self, name, range=None):
propertyData = PropertyData(name)
self.allPropertyData.append(propertyData)
self.propertyDataByCategory[name] = propertyData
if range:
propertyData.addRange(range[0], range[1])
return propertyData
def findPropertyGroupFor(self, categoryAlias):
category = aliases.generalCategoryForAlias(categoryAlias)
allCategoryAliases = aliases.generalCategoryAliasesFor(category)
categoryGroupAlias = categoryAlias[0]
categoryGroup = aliases.generalCategoryForAlias(categoryGroupAlias)
allCategoryGroupAlias = aliases.generalCategoryAliasesFor(categoryGroup)
groupPropertyData = None
propertyData = None
if categoryGroup not in self.propertyDataByCategory:
groupPropertyData = PropertyData(categoryGroup)
groupPropertyData.setAliases(allCategoryGroupAlias)
self.allPropertyData.append(groupPropertyData)
self.propertyDataByCategory[categoryGroup] = groupPropertyData
else:
groupPropertyData = self.propertyDataByCategory[categoryGroup]
if category not in self.propertyDataByCategory:
propertyData = PropertyData(category)
propertyData.setAliases(allCategoryAliases)
self.allPropertyData.append(propertyData)
self.propertyDataByCategory[category] = propertyData
else:
propertyData = self.propertyDataByCategory[category]
return (groupPropertyData, propertyData)
def addNextCodePoints(self, categoryAlias, codePoint, highCodePoint=None):
if codePoint - self.lastAddedCodePoint > 1:
propertyDatas = self.findPropertyGroupFor("Cn")
if codePoint - self.lastAddedCodePoint == 2:
propertyDatas[0].addMatch(self.lastAddedCodePoint + 1)
propertyDatas[1].addMatch(self.lastAddedCodePoint + 1)
else:
propertyDatas[0].addRange(self.lastAddedCodePoint + 1, codePoint - 1)
propertyDatas[1].addRange(self.lastAddedCodePoint + 1, codePoint - 1)
propertyDatas = self.findPropertyGroupFor(categoryAlias)
if highCodePoint:
propertyDatas[0].addRange(codePoint, highCodePoint)
propertyDatas[1].addRange(codePoint, highCodePoint)
if categoryAlias == "Ll" or categoryAlias == "Lt" or categoryAlias == "Lu":
self.casedLetterPropertyData.addRange(codePoint, highCodePoint)
self.assignedPropertyData.addRange(codePoint, highCodePoint)
self.lastAddedCodePoint = highCodePoint
else:
propertyDatas[0].addMatch(codePoint)
propertyDatas[1].addMatch(codePoint)
if categoryAlias == "Ll" or categoryAlias == "Lt" or categoryAlias == "Lu":
self.casedLetterPropertyData.addMatch(codePoint)
self.assignedPropertyData.addMatch(codePoint)
self.lastAddedCodePoint = codePoint
def parse(self):
lastLineFirstOfRange = None
lastLineCodePoint = 0
for line in self.file:
line = line.split('#', 1)[0]
line = line.rstrip()
if (not len(line)):
continue
fields = line.split(';')
if (not fields):
continue
codePoint = int(fields[0].strip(), 16)
description = fields[1].strip()
categoryAlias = fields[2].strip()
if lastLineFirstOfRange:
if description[-5:-1] == "Last":
self.addNextCodePoints(categoryAlias, lastLineFirstOfRange, codePoint)
lastLineFirstOfRange = None
continue
else:
print("Malformed First..Last pair of lines")
if description[-6:-1] == "First":
lastLineFirstOfRange = codePoint
continue
self.addNextCodePoints(categoryAlias, codePoint)
if self.lastAddedCodePoint < MaxUnicode:
propertyDatas = self.findPropertyGroupFor("Cn")
if MaxUnicode - self.lastAddedCodePoint == 1:
propertyDatas[0].addMatch(MaxUnicode)
propertyDatas[1].addMatch(MaxUnicode)
else:
propertyDatas[0].addRange(self.lastAddedCodePoint + 1, MaxUnicode)
propertyDatas[1].addRange(self.lastAddedCodePoint + 1, MaxUnicode)
def dump(self, file):
file.write("// General_Category:\n")
PropertyData.createAndDumpHashTable(file, self.propertyDataByCategory, "generalCategory")
class BinaryProperty:
def __init__(self):
self.allPropertyData = []
self.propertyDataByProperty = {}
def parsePropertyFile(self, file):
currentPropertyName = None
currentPropertyData = None
for line in file:
line = line.split('#', 1)[0]
line = line.rstrip()
if (not len(line)):
continue
fields = line.split(';')
if (not fields):
continue
codePoints = fields[0].strip()
propertyName = fields[1].strip()
if propertyName != currentPropertyName:
if propertyName not in SupportedBinaryProperties:
continue
currentPropertyName = propertyName
currentPropertyData = PropertyData(propertyName)
currentPropertyData.setAliases(aliases.globalAliasesFor(propertyName))
self.allPropertyData.append(currentPropertyData)
self.propertyDataByProperty[propertyName] = currentPropertyData
dotDot = codePoints.find("..")
if dotDot == -1:
currentPropertyData.addMatch(int(codePoints, 16))
else:
currentPropertyData.addRange(int(codePoints[:dotDot], 16), int(codePoints[dotDot + 2:], 16))
def dump(self, file):
file.write("// binary properties:\n")
PropertyData.createAndDumpHashTable(file, self.propertyDataByProperty, "binaryProperty")
if __name__ == "__main__":
parser = optparse.OptionParser(usage="usage: %prog <UCD-Directory> <YarrUnicodePropertyData.h>")
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error("<UCD-Directory> <YarrUnicodePropertyData.h>")
UCDDirectoryPath = args[0]
unicodeProertyDataHPath = args[1]
verifyUCDFilesExist()
propertyAliasesFile = openUCDFileOrExit("PropertyAliases.txt")
propertyValueAliasesFile = openUCDFileOrExit("PropertyValueAliases.txt")
scriptsFile = openUCDFileOrExit("Scripts.txt")
scriptExtensionsFile = openUCDFileOrExit("ScriptExtensions.txt")
unicodeDataFile = openUCDFileOrExit("UnicodeData.txt")
derivedBinaryPropertiesFile = openUCDFileOrExit("DerivedBinaryProperties.txt")
derivedCorePropertiesFile = openUCDFileOrExit("DerivedCoreProperties.txt")
derivedNormalizationPropertiesFile = openUCDFileOrExit("DerivedNormalizationProps.txt")
propListFile = openUCDFileOrExit("PropList.txt")
emojiDataFile = openUCDFileOrExit("emoji-data.txt")
aliases = Aliases()
propertyDataHFile = openOrExit(unicodeProertyDataHPath, "wb")
propertyDataHFile.write(header)
aliases.parsePropertyAliasesFile(propertyAliasesFile)
aliases.parsePropertyValueAliasesFile(propertyValueAliasesFile)
generalCategory = GeneralCategory(unicodeDataFile)
generalCategory.parse()
binaryProperty = BinaryProperty()
binaryProperty.parsePropertyFile(derivedBinaryPropertiesFile)
binaryProperty.parsePropertyFile(derivedCorePropertiesFile)
binaryProperty.parsePropertyFile(derivedNormalizationPropertiesFile)
binaryProperty.parsePropertyFile(propListFile)
binaryProperty.parsePropertyFile(emojiDataFile)
scripts = Scripts()
scripts.parseScriptsFile(scriptsFile)
scripts.parseScriptExtensionsFile(scriptExtensionsFile)
PropertyData.dumpAll(propertyDataHFile)
generalCategory.dump(propertyDataHFile)
binaryProperty.dump(propertyDataHFile)
scripts.dump(propertyDataHFile)
propertyDataHFile.write(footer)
exit(0)