unless ARGV.size == 1
$stderr.puts "Usage: #{$0} data_directory"
exit(1)
end
POSIX_NAMES = %w[NEWLINE Alpha Blank Cntrl Digit Graph Lower Print Punct Space Upper XDigit Word Alnum ASCII]
def pair_codepoints(codepoints)
codepoints.sort!
last_cp = codepoints.first
pairs = [[last_cp, nil]]
codepoints[1..-1].each do |codepoint|
next if last_cp == codepoint
if last_cp.next != codepoint
pairs[-1][-1] = last_cp
pairs << [codepoint, nil]
end
last_cp = codepoint
end
pairs[-1][-1] = codepoints.last
pairs
end
def parse_unicode_data(file)
last_cp = 0
data = {'Any' => (0x0000..0x10ffff).to_a, 'Assigned' => [],
'ASCII' => (0..0x007F).to_a, 'NEWLINE' => [0x0a], 'Cn' => []}
beg_cp = nil
IO.foreach(file) do |line|
fields = line.split(';')
cp = fields[0].to_i(16)
case fields[1]
when /\A<(.*),\s*First>\z/
beg_cp = cp
next
when /\A<(.*),\s*Last>\z/
cps = (beg_cp..cp).to_a
else
beg_cp = cp
cps = [cp]
end
data['Cn'].concat((last_cp.next...beg_cp).to_a)
data['Assigned'].concat(cps)
(data[fields[2]] ||= []).concat(cps)
(data[fields[2][0,1]] ||= []).concat(cps)
last_cp = cp
end
cn_remainder = (last_cp.next..0x10ffff).to_a
data['Cn'] += cn_remainder
data['C'] += data['Cn']
data['LC'] = data['Ll'] + data['Lt'] + data['Lu']
gcps = data.keys.sort - POSIX_NAMES
[gcps, data]
end
def define_posix_props(data)
data['Alpha'] = data['Alphabetic']
data['Upper'] = data['Uppercase']
data['Lower'] = data['Lowercase']
data['Punct'] = data['Punctuation']
data['Digit'] = data['Decimal_Number']
data['XDigit'] = (0x0030..0x0039).to_a + (0x0041..0x0046).to_a +
(0x0061..0x0066).to_a
data['Alnum'] = data['Alpha'] + data['Digit']
data['Space'] = data['White_Space']
data['Blank'] = data['Space_Separator'] + [0x0009]
data['Cntrl'] = data['Cc']
data['Word'] = data['Alpha'] + data['Mark'] + data['Digit'] + data['Connector_Punctuation']
data['Graph'] = data['Any'] - data['Space'] - data['Cntrl'] -
data['Surrogate'] - data['Unassigned']
data['Print'] = data['Graph'] + data['Space_Separator']
end
def parse_scripts(data, categories)
files = [
{:fn => 'DerivedCoreProperties.txt', :title => 'Derived Property'},
{:fn => 'Scripts.txt', :title => 'Script'},
{:fn => 'PropList.txt', :title => 'Binary Property'}
]
current = nil
cps = []
names = {}
files.each do |file|
IO.foreach(get_file(file[:fn])) do |line|
if /^ data[current] = cps
categories[current] = file[:title]
(names[file[:title]] ||= []) << current
cps = []
elsif /^([0-9a-fA-F]+)(?:..([0-9a-fA-F]+))?\s*;\s*(\w+)/ =~ line
current = $3
$2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16))
end
end
end
data['Unknown'] = (0..0x10ffff).to_a - data.values_at(*names['Script']).flatten
categories['Unknown'] = 'Script'
names.values.flatten << 'Unknown'
end
def parse_aliases(data)
kv = {}
IO.foreach(get_file('PropertyAliases.txt')) do |line|
next unless /^(\w+)\s*; (\w+)/ =~ line
data[$1] = data[$2]
kv[normalize_propname($1)] = normalize_propname($2)
end
IO.foreach(get_file('PropertyValueAliases.txt')) do |line|
next unless /^(sc|gc)\s*; (\w+)\s*; (\w+)(?:\s*; (\w+))?/ =~ line
if $1 == 'gc'
data[$3] = data[$2]
data[$4] = data[$2]
kv[normalize_propname($3)] = normalize_propname($2)
kv[normalize_propname($4)] = normalize_propname($2) if $4
else
data[$2] = data[$3]
data[$4] = data[$3]
kv[normalize_propname($2)] = normalize_propname($3)
kv[normalize_propname($4)] = normalize_propname($3) if $4
end
end
kv
end
def parse_age(data)
current = nil
last_constname = nil
cps = []
ages = []
IO.foreach(get_file('DerivedAge.txt')) do |line|
if /^ constname = constantize_agename(current)
cps.concat(data[last_constname]) if last_constname
data[constname] = cps
make_const(constname, cps, "Derived Age #{current}")
ages << current
last_constname = constname
cps = []
elsif /^([0-9a-fA-F]+)(?:..([0-9a-fA-F]+))?\s*;\s*(\d+\.\d+)/ =~ line
current = $3
$2 ? cps.concat(($1.to_i(16)..$2.to_i(16)).to_a) : cps.push($1.to_i(16))
end
end
ages
end
def parse_block(data)
current = nil
last_constname = nil
cps = []
blocks = []
IO.foreach(get_file('Blocks.txt')) do |line|
if /^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+);\s*(.*)/ =~ line
cps = ($1.to_i(16)..$2.to_i(16)).to_a
constname = constantize_blockname($3)
data[constname] = cps
make_const(constname, cps, "Block")
blocks << constname
end
end
no_block = (0..0x10ffff).to_a - data.values_at(*blocks).flatten
constname = constantize_blockname("No_Block")
make_const(constname, no_block, "Block")
blocks << constname
end
unless {}.respond_to?(:key)
class Hash
alias key index
end
end
$const_cache = {}
def make_const(prop, data, name)
puts "\n/* '#{prop}': #{name} */"
if origprop = $const_cache.key(data)
puts "#define CR_#{prop} CR_#{origprop}"
else
$const_cache[prop] = data
pairs = pair_codepoints(data)
puts "static const OnigCodePoint CR_#{prop}[] = {"
puts "\t#{pairs.size},"
pairs.each do |pair|
pair.map! { |c| c == 0 ? '0x0000' : sprintf("%0#6x", c) }
puts "\t#{pair.first}, #{pair.last},"
end
puts "}; /* CR_#{prop} */"
end
end
def normalize_propname(name)
name = name.downcase
name.delete!('- _')
name
end
def constantize_agename(name)
"Age_#{name.sub(/\./, '_')}"
end
def constantize_blockname(name)
"In_#{name.gsub(/\W/, '_')}"
end
def get_file(name)
File.join(ARGV[0], name)
end
puts '%{'
puts '#define long size_t'
props, data = parse_unicode_data(get_file('UnicodeData.txt'))
categories = {}
props.concat parse_scripts(data, categories)
aliases = parse_aliases(data)
define_posix_props(data)
POSIX_NAMES.each do |name|
make_const(name, data[name], "[[:#{name}:]]")
end
print "\n#ifdef USE_UNICODE_PROPERTIES"
props.each do |name|
category = categories[name] ||
case name.size
when 1 then 'Major Category'
when 2 then 'General Category'
else '-'
end
make_const(name, data[name], category)
end
ages = parse_age(data)
blocks = parse_block(data)
puts '#endif /* USE_UNICODE_PROPERTIES */'
puts(<<'__HEREDOC')
static const OnigCodePoint* const CodeRanges[] = {
__HEREDOC
POSIX_NAMES.each{|name|puts" CR_#{name},"}
puts "#ifdef USE_UNICODE_PROPERTIES"
props.each{|name| puts" CR_#{name},"}
ages.each{|name| puts" CR_#{constantize_agename(name)},"}
blocks.each{|name|puts" CR_#{name},"}
puts(<<'__HEREDOC')
};
struct uniname2ctype_struct {
int name, ctype;
};
static const struct uniname2ctype_struct *uniname2ctype_p(const char *, unsigned int);
%}
struct uniname2ctype_struct;
%%
__HEREDOC
i = -1
name_to_index = {}
POSIX_NAMES.each do |name|
i += 1
next if name == 'NEWLINE'
name = normalize_propname(name)
name_to_index[name] = i
puts"%-40s %3d" % [name + ',', i]
end
puts "#ifdef USE_UNICODE_PROPERTIES"
props.each do |name|
i += 1
name = normalize_propname(name)
name_to_index[name] = i
puts "%-40s %3d" % [name + ',', i]
end
aliases.each_pair do |k, v|
next if name_to_index[k]
next unless v = name_to_index[v]
puts "%-40s %3d" % [k + ',', v]
end
ages.each do |name|
i += 1
name = "age=#{name}"
name_to_index[name] = i
puts "%-40s %3d" % [name + ',', i]
end
blocks.each do |name|
i += 1
name = normalize_propname(name)
name_to_index[name] = i
puts "%-40s %3d" % [name + ',', i]
end
puts(<<'__HEREDOC')
%%
static int
uniname2ctype(const UChar *name, unsigned int len)
{
const struct uniname2ctype_struct *p = uniname2ctype_p((const char *)name, len);
if (p) return p->ctype;
return -1;
}
__HEREDOC