Commit d67d49ce authored by Glenn Morris's avatar Glenn Morris
Browse files

Generate char-script-table from Unicode source. (Bug#20789)

* admin/unidata/ (AWK): New, set by configure.
(all): Add charscript.el.
(blocks): New variable.
(charscript.el, ${unidir}/charscript.el): New targets.
(extraclean): Also remove generated charscript.el.

* admin/unidata/blocks.awk: New script.

* admin/unidata/Blocks.txt: New data file, from

* lisp/international/characters.el: Load charscript.

* src/ (charscript): New variable.
(${charscript}): New target.
(${lispintdir}/characters.elc): Depend on charscript.elc.
(temacs$(EXEEXT)): Depend on charscript.

; * admin/unidata/README: Mention Blocks.txt.

; * .gitignore: Add lisp/international/charscript.el.
parent c9e2003b
......@@ -195,6 +195,7 @@ admin/charsets/jisx2131-filter
# Blocks-8.0.0.txt
# Date: 2014-11-10, 23:04:00 GMT [KW]
# Unicode Character Database
# Copyright (c) 1991-2014 Unicode, Inc.
# For terms of use, see
# For documentation, see
# Format:
# Start Code..End Code; Block Name
# ================================================
# Note: When comparing block names, casing, whitespace, hyphens,
# and underbars are ignored.
# For example, "Latin Extended-A" and "latin extended a" are equivalent.
# For more information on the comparison of property values,
# see UAX #44:
# All block ranges start with a value where (cp MOD 16) = 0,
# and end with a value where (cp MOD 16) = 15. In other words,
# the last hexadecimal digit of the start of range is ...0
# and the last hexadecimal digit of the end of range is ...F.
# This constraint on block ranges guarantees that allocations
# are done in terms of whole columns, and that code chart display
# never involves splitting columns in the charts.
# All code points not explicitly listed for Block
# have the value No_Block.
# Property: Block
# @missing: 0000..10FFFF; No_Block
0000..007F; Basic Latin
0080..00FF; Latin-1 Supplement
0100..017F; Latin Extended-A
0180..024F; Latin Extended-B
0250..02AF; IPA Extensions
02B0..02FF; Spacing Modifier Letters
0300..036F; Combining Diacritical Marks
0370..03FF; Greek and Coptic
0400..04FF; Cyrillic
0500..052F; Cyrillic Supplement
0530..058F; Armenian
0590..05FF; Hebrew
0600..06FF; Arabic
0700..074F; Syriac
0750..077F; Arabic Supplement
0780..07BF; Thaana
07C0..07FF; NKo
0800..083F; Samaritan
0840..085F; Mandaic
08A0..08FF; Arabic Extended-A
0900..097F; Devanagari
0980..09FF; Bengali
0A00..0A7F; Gurmukhi
0A80..0AFF; Gujarati
0B00..0B7F; Oriya
0B80..0BFF; Tamil
0C00..0C7F; Telugu
0C80..0CFF; Kannada
0D00..0D7F; Malayalam
0D80..0DFF; Sinhala
0E00..0E7F; Thai
0E80..0EFF; Lao
0F00..0FFF; Tibetan
1000..109F; Myanmar
10A0..10FF; Georgian
1100..11FF; Hangul Jamo
1200..137F; Ethiopic
1380..139F; Ethiopic Supplement
13A0..13FF; Cherokee
1400..167F; Unified Canadian Aboriginal Syllabics
1680..169F; Ogham
16A0..16FF; Runic
1700..171F; Tagalog
1720..173F; Hanunoo
1740..175F; Buhid
1760..177F; Tagbanwa
1780..17FF; Khmer
1800..18AF; Mongolian
18B0..18FF; Unified Canadian Aboriginal Syllabics Extended
1900..194F; Limbu
1950..197F; Tai Le
1980..19DF; New Tai Lue
19E0..19FF; Khmer Symbols
1A00..1A1F; Buginese
1A20..1AAF; Tai Tham
1AB0..1AFF; Combining Diacritical Marks Extended
1B00..1B7F; Balinese
1B80..1BBF; Sundanese
1BC0..1BFF; Batak
1C00..1C4F; Lepcha
1C50..1C7F; Ol Chiki
1CC0..1CCF; Sundanese Supplement
1CD0..1CFF; Vedic Extensions
1D00..1D7F; Phonetic Extensions
1D80..1DBF; Phonetic Extensions Supplement
1DC0..1DFF; Combining Diacritical Marks Supplement
1E00..1EFF; Latin Extended Additional
1F00..1FFF; Greek Extended
2000..206F; General Punctuation
2070..209F; Superscripts and Subscripts
20A0..20CF; Currency Symbols
20D0..20FF; Combining Diacritical Marks for Symbols
2100..214F; Letterlike Symbols
2150..218F; Number Forms
2190..21FF; Arrows
2200..22FF; Mathematical Operators
2300..23FF; Miscellaneous Technical
2400..243F; Control Pictures
2440..245F; Optical Character Recognition
2460..24FF; Enclosed Alphanumerics
2500..257F; Box Drawing
2580..259F; Block Elements
25A0..25FF; Geometric Shapes
2600..26FF; Miscellaneous Symbols
2700..27BF; Dingbats
27C0..27EF; Miscellaneous Mathematical Symbols-A
27F0..27FF; Supplemental Arrows-A
2800..28FF; Braille Patterns
2900..297F; Supplemental Arrows-B
2980..29FF; Miscellaneous Mathematical Symbols-B
2A00..2AFF; Supplemental Mathematical Operators
2B00..2BFF; Miscellaneous Symbols and Arrows
2C00..2C5F; Glagolitic
2C60..2C7F; Latin Extended-C
2C80..2CFF; Coptic
2D00..2D2F; Georgian Supplement
2D30..2D7F; Tifinagh
2D80..2DDF; Ethiopic Extended
2DE0..2DFF; Cyrillic Extended-A
2E00..2E7F; Supplemental Punctuation
2E80..2EFF; CJK Radicals Supplement
2F00..2FDF; Kangxi Radicals
2FF0..2FFF; Ideographic Description Characters
3000..303F; CJK Symbols and Punctuation
3040..309F; Hiragana
30A0..30FF; Katakana
3100..312F; Bopomofo
3130..318F; Hangul Compatibility Jamo
3190..319F; Kanbun
31A0..31BF; Bopomofo Extended
31C0..31EF; CJK Strokes
31F0..31FF; Katakana Phonetic Extensions
3200..32FF; Enclosed CJK Letters and Months
3300..33FF; CJK Compatibility
3400..4DBF; CJK Unified Ideographs Extension A
4DC0..4DFF; Yijing Hexagram Symbols
4E00..9FFF; CJK Unified Ideographs
A000..A48F; Yi Syllables
A490..A4CF; Yi Radicals
A4D0..A4FF; Lisu
A500..A63F; Vai
A640..A69F; Cyrillic Extended-B
A6A0..A6FF; Bamum
A700..A71F; Modifier Tone Letters
A720..A7FF; Latin Extended-D
A800..A82F; Syloti Nagri
A830..A83F; Common Indic Number Forms
A840..A87F; Phags-pa
A880..A8DF; Saurashtra
A8E0..A8FF; Devanagari Extended
A900..A92F; Kayah Li
A930..A95F; Rejang
A960..A97F; Hangul Jamo Extended-A
A980..A9DF; Javanese
A9E0..A9FF; Myanmar Extended-B
AA00..AA5F; Cham
AA60..AA7F; Myanmar Extended-A
AA80..AADF; Tai Viet
AAE0..AAFF; Meetei Mayek Extensions
AB00..AB2F; Ethiopic Extended-A
AB30..AB6F; Latin Extended-E
AB70..ABBF; Cherokee Supplement
ABC0..ABFF; Meetei Mayek
AC00..D7AF; Hangul Syllables
D7B0..D7FF; Hangul Jamo Extended-B
D800..DB7F; High Surrogates
DB80..DBFF; High Private Use Surrogates
DC00..DFFF; Low Surrogates
E000..F8FF; Private Use Area
F900..FAFF; CJK Compatibility Ideographs
FB00..FB4F; Alphabetic Presentation Forms
FB50..FDFF; Arabic Presentation Forms-A
FE00..FE0F; Variation Selectors
FE10..FE1F; Vertical Forms
FE20..FE2F; Combining Half Marks
FE30..FE4F; CJK Compatibility Forms
FE50..FE6F; Small Form Variants
FE70..FEFF; Arabic Presentation Forms-B
FF00..FFEF; Halfwidth and Fullwidth Forms
FFF0..FFFF; Specials
10000..1007F; Linear B Syllabary
10080..100FF; Linear B Ideograms
10100..1013F; Aegean Numbers
10140..1018F; Ancient Greek Numbers
10190..101CF; Ancient Symbols
101D0..101FF; Phaistos Disc
10280..1029F; Lycian
102A0..102DF; Carian
102E0..102FF; Coptic Epact Numbers
10300..1032F; Old Italic
10330..1034F; Gothic
10350..1037F; Old Permic
10380..1039F; Ugaritic
103A0..103DF; Old Persian
10400..1044F; Deseret
10450..1047F; Shavian
10480..104AF; Osmanya
10500..1052F; Elbasan
10530..1056F; Caucasian Albanian
10600..1077F; Linear A
10800..1083F; Cypriot Syllabary
10840..1085F; Imperial Aramaic
10860..1087F; Palmyrene
10880..108AF; Nabataean
108E0..108FF; Hatran
10900..1091F; Phoenician
10920..1093F; Lydian
10980..1099F; Meroitic Hieroglyphs
109A0..109FF; Meroitic Cursive
10A00..10A5F; Kharoshthi
10A60..10A7F; Old South Arabian
10A80..10A9F; Old North Arabian
10AC0..10AFF; Manichaean
10B00..10B3F; Avestan
10B40..10B5F; Inscriptional Parthian
10B60..10B7F; Inscriptional Pahlavi
10B80..10BAF; Psalter Pahlavi
10C00..10C4F; Old Turkic
10C80..10CFF; Old Hungarian
10E60..10E7F; Rumi Numeral Symbols
11000..1107F; Brahmi
11080..110CF; Kaithi
110D0..110FF; Sora Sompeng
11100..1114F; Chakma
11150..1117F; Mahajani
11180..111DF; Sharada
111E0..111FF; Sinhala Archaic Numbers
11200..1124F; Khojki
11280..112AF; Multani
112B0..112FF; Khudawadi
11300..1137F; Grantha
11480..114DF; Tirhuta
11580..115FF; Siddham
11600..1165F; Modi
11680..116CF; Takri
11700..1173F; Ahom
118A0..118FF; Warang Citi
11AC0..11AFF; Pau Cin Hau
12000..123FF; Cuneiform
12400..1247F; Cuneiform Numbers and Punctuation
12480..1254F; Early Dynastic Cuneiform
13000..1342F; Egyptian Hieroglyphs
14400..1467F; Anatolian Hieroglyphs
16800..16A3F; Bamum Supplement
16A40..16A6F; Mro
16AD0..16AFF; Bassa Vah
16B00..16B8F; Pahawh Hmong
16F00..16F9F; Miao
1B000..1B0FF; Kana Supplement
1BC00..1BC9F; Duployan
1BCA0..1BCAF; Shorthand Format Controls
1D000..1D0FF; Byzantine Musical Symbols
1D100..1D1FF; Musical Symbols
1D200..1D24F; Ancient Greek Musical Notation
1D300..1D35F; Tai Xuan Jing Symbols
1D360..1D37F; Counting Rod Numerals
1D400..1D7FF; Mathematical Alphanumeric Symbols
1D800..1DAAF; Sutton SignWriting
1E800..1E8DF; Mende Kikakui
1EE00..1EEFF; Arabic Mathematical Alphabetic Symbols
1F000..1F02F; Mahjong Tiles
1F030..1F09F; Domino Tiles
1F0A0..1F0FF; Playing Cards
1F100..1F1FF; Enclosed Alphanumeric Supplement
1F200..1F2FF; Enclosed Ideographic Supplement
1F300..1F5FF; Miscellaneous Symbols and Pictographs
1F600..1F64F; Emoticons
1F650..1F67F; Ornamental Dingbats
1F680..1F6FF; Transport and Map Symbols
1F700..1F77F; Alchemical Symbols
1F780..1F7FF; Geometric Shapes Extended
1F800..1F8FF; Supplemental Arrows-C
1F900..1F9FF; Supplemental Symbols and Pictographs
20000..2A6DF; CJK Unified Ideographs Extension B
2A700..2B73F; CJK Unified Ideographs Extension C
2B740..2B81F; CJK Unified Ideographs Extension D
2B820..2CEAF; CJK Unified Ideographs Extension E
2F800..2FA1F; CJK Compatibility Ideographs Supplement
E0000..E007F; Tags
E0100..E01EF; Variation Selectors Supplement
F0000..FFFFF; Supplementary Private Use Area-A
100000..10FFFF; Supplementary Private Use Area-B
......@@ -23,6 +23,8 @@
srcdir = @srcdir@
top_srcdir = @top_srcdir@
top_builddir = @top_builddir@
......@@ -51,7 +53,7 @@ am__v_at_1 =
.PHONY: all unifiles
all: ${top_srcdir}/src/macuvs.h unifiles
all: ${top_srcdir}/src/macuvs.h unifiles ${unidir}/charscript.el
## Specify .elc as an order-only prereq so as to not needlessly rebuild
## target just because the .elc is missing.
......@@ -94,6 +96,16 @@ unifiles: ${unidir}/charprop.el
[ -f $(unidir)/$$f ] || exec $(MAKE) PHONY_EXTRAS=$< $<; \
.PHONY: charscript.el
charscript.el: ${unidir}/charscript.el
blocks = ${srcdir}/blocks.awk
${unidir}/charscript.el: ${srcdir}/Blocks.txt ${blocks}
$(AM_V_GEN)$(AWK) -f ${blocks} < $< > $@
.PHONY: clean bootstrap-clean distclean maintainer-clean extraclean
......@@ -110,7 +122,7 @@ maintainer-clean: distclean
## change and it slows down bootstrap (a tiny bit).
## Cf leim/ja-dic (which is much slower).
rm -f ${top_srcdir}/src/macuvs.h
rm -f ${top_srcdir}/src/macuvs.h ${unidir}/charscript.el*
ifneq (,$(wildcard $(unidir)/charprop.el))
cd $(unidir) && \
rm -f `sed -n 's/^;; FILE: //p' < charprop.el` charprop.el
......@@ -16,3 +16,7 @@
#!/usr/bin/awk -f
## Copyright (C) 2015 Free Software Foundation, Inc.
## Author: Glenn Morris <>
## This file is part of GNU Emacs.
## GNU Emacs is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
## GNU Emacs is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## GNU General Public License for more details.
## You should have received a copy of the GNU General Public License
## along with GNU Emacs. If not, see <>.
### Commentary:
## This script takes as input Unicode's Blocks.txt
## (
## and produces output for Emacs's lisp/international/charscript.el.
## It lumps together all the blocks belonging to the same language.
## E.g., "Basic Latin", "Latin-1 Supplement", "Latin Extended-A",
## etc. are all lumped together under "latin".
## The Unicode blocks actually extend past some of these ranges with
## undefined codepoints.
## For additional details, see <>.
## Things to do after installing a new version of Blocks.txt:
## Check the output against the old output.
## Adjust the alias array, and the name2alias function for any new
## entries, if necessary.
## Check fix_start (and fix_end) to see if entries need adding/removing.
## Review the hard-coded splits at the end of the main body.
### Code:
## Hard-coded names. See name2alias for the rest.
alias["ipa extensions"] = "phonetic"
alias["letterlike symbols"] = "symbol"
alias["number forms"] = "symbol"
alias["miscellaneous technical"] = "symbol"
alias["control pictures"] = "symbol"
alias["optical character recognition"] = "symbol"
alias["enclosed alphanumerics"] = "symbol"
alias["box drawing"] = "symbol"
alias["block elements"] = "symbol"
alias["miscellaneous symbols"] = "symbol"
alias["cjk strokes"] = "cjk-misc"
alias["cjk symbols and punctuation"] = "cjk-misc"
alias["halfwidth and fullwidth forms"] = "cjk-misc"
alias["common indic number forms"] = "north-indic-number"
tohex["a"] = 10
tohex["b"] = 11
tohex["c"] = 12
tohex["d"] = 13
tohex["e"] = 14
tohex["f"] = 15
fix_start["0080"] = "00A0"
## Define fix_end here if you need it.
## From admin/charsets/.
## With gawk's --non-decimal-data switch we wouldn't need this.
function decode_hex(str , n, len, i, c) {
n = 0
len = length(str)
for (i = 1; i <= len; i++)
c = substr (str, i, 1)
if (c >= "0" && c <= "9")
n = n * 16 + (c - "0")
n = n * 16 + tohex[tolower(c)]
return n
function name2alias(name , w, w2) {
name = tolower(name)
if (alias[name]) return alias[name]
else if (name ~ /for symbols/) return "symbol"
else if (name ~ /latin|combining .* marks|spacing modifier|tone letters|alphabetic presentation/) return "latin"
else if (name ~ /cjk|yijing|enclosed ideograph|kangxi/) return "han"
else if (name ~ /arabic/) return "arabic"
else if (name ~ /^greek/) return "greek"
else if (name ~ /^coptic/) return "coptic"
else if (name ~ /cuneiform number/) return "cuneiform-numbers-and-punctuation"
else if (name ~ /cuneiform/) return "cuneiform"
else if (name ~ /mathematical alphanumeric symbol/) return "mathematical"
else if (name ~ /punctuation|mathematical|arrows|currency|superscript|small form variants|geometric|dingbats|enclosed|alchemical|pictograph|emoticon|transport/) return "symbol"
else if (name ~ /canadian aboriginal/) return "canadian-aboriginal"
else if (name ~ /katakana|hiragana/) return "kana"
else if (name ~ /myanmar/) return "burmese"
else if (name ~ /hangul/) return "hangul"
else if (name ~ /khmer/) return "khmer"
else if (name ~ /braille/) return "braille"
else if (name ~ /^yi /) return "yi"
else if (name ~ /surrogates|private use|variation selectors/) return 0
else if (name ~/^(specials|tags)$/) return 0
else if (name ~ /linear b/) return "linear-b"
else if (name ~ /aramaic/) return "aramaic"
else if (name ~ /rumi num/) return "rumi-number"
else if (name ~ /duployan|shorthand/) return "duployan-shorthand"
else if (name ~ /sutton signwriting/) return "sutton-sign-writing"
sub(/ (extended|extensions|supplement).*/, "", name)
sub(/numbers/, "number", name)
sub(/numerals/, "numeral", name)
sub(/symbols/, "symbol", name)
sub(/forms$/, "form", name)
sub(/tiles$/, "tile", name)
sub(/^new /, "", name)
sub(/ (characters|hieroglyphs|cursive)$/, "", name)
gsub(/ /, "-", name)
return name
/^[0-9A-F]/ {
sep = index($1, "..")
len = length($1)
s = substr($1,1,sep-1)
e = substr($1,sep+2,len-sep-2)
$1 = ""
sub(/^ */, "", $0)
start[i] = fix_start[s] ? fix_start[s] : s
end[i] = fix_end[e] ? fix_end[e]: e
name[i] = $0
alt[i] = name2alias(name[i])
if (!alt[i])
## Combine adjacent ranges with the same name.
if (alt[i] == alt[i-1] && decode_hex(start[i]) == 1 + decode_hex(end[i-1]))
end[i-1] = end[i]
name[i-1] = (name[i-1] ", " name[i])
## Some hard-coded splits.
if (start[i] == "0370")
end[i] = "03E1"
start[i] = "03E2"
end[i] = "03EF"
alt[i] = "coptic"
start[i] = "03F0"
end[i] = "03FF"
alt[i] = "greek"
else if (start[i] == "FB00")
end[i] = "FB06"
start[i] = "FB13"
end[i] = "FB17"
alt[i] = "armenian"
start[i] = "FB1D"
end[i] = "FB4F"
alt[i] = "hebrew"
else if (start[i] == "FF00")
end[i] = "FF60"
start[i] = "FF61"
end[i] = "FF9F"
alt[i] = "kana"
start[i] = "FFA0"
end[i] = "FFDF"
alt[i] = "hangul"
start[i] = "FFE0"
end[i] = "FFEF"
alt[i] = "cjk-misc"
print ";;; charscript.el --- character script table"
print ";;; Automatically generated from admin/unidata/Blocks.txt"
print "(let (script-list)"
print " (dolist (elt '("
for (j=1;j<=i;j++)
printf(" (#x%s #x%s %s)", start[j], end[j], alt[j])
## Fuzz to decide whether worth printing original name as a comment.
if (name[j] && alt[j] != tolower(name[j]) && alt[j] !~ /-/)
printf(" ; %s", name[j])
print " ))"
print " (set-char-table-range char-script-table"
print " (cons (car elt) (nth 1 elt)) (nth 2 elt))"
print " (or (memq (nth 2 elt) script-list)"
print " (setq script-list (cons (nth 2 elt) script-list))))"
print " (set-char-table-extra-slot char-script-table 0 (nreverse script-list)))"
print ""
print "(provide 'charscript)"
......@@ -1163,235 +1163,11 @@ Setup char-width-table appropriate for non-CJK language environment."
;; Setting char-script-table.
;; The data is compiled from Blocks.txt and Scripts.txt in the
;; "Unicode Character Database", simplified to lump together all the
;; blocks belonging to the same language. E.g., "Basic Latin",
;; "Latin-1 Supplement", "Latin Extended-A", etc. are all lumped
;; together under "latin".
;; The Unicode blocks actually extend past some of these ranges with
;; undefined codepoints.
;; Last update:
(let ((script-list nil))
'((#x0000 #x007F latin)
(#x00A0 #x024F latin)
(#x0250 #x02AF phonetic) ; IPA Extensions
(#x02B0 #x036F latin) ; Spacing Modifiers and Diacriticals
(#x0370 #x03E1 greek)
(#x03E2 #x03EF coptic)
(#x03F0 #x03FF greek)
(#x0400 #x052F cyrillic)
(#x0530 #x058F armenian)
(#x0590 #x05FF hebrew)
(#x0600 #x06FF arabic)
(#x0700 #x074F syriac)
(#x0750 #x077F arabic) ; Arabic Supplement
(#x0780 #x07BF thaana)
(#x07C0 #x07FF nko)
(#x0800 #x083F samaritan)
(#x0840 #x085F mandaic)
(#x08A0 #x08FF arabic) ; Arabic Extended-A
(#x0900 #x097F devanagari)
(#x0980 #x09FF bengali)
(#x0A00 #x0A7F gurmukhi)
(#x0A80 #x0AFF gujarati)
(#x0B00 #x0B7F oriya)
(#x0B80 #x0BFF tamil)
(#x0C00 #x0C7F telugu)
(#x0C80 #x0CFF kannada)
(#x0D00 #x0D7F malayalam)
(#x0D80 #x0DFF sinhala)
(#x0E00 #x0E7F thai)
(#x0E80 #x0EFF lao)
(#x0F00 #x0FFF tibetan)
(#x1000 #x109F burmese) ; Myanmar
(#x10A0 #x10FF georgian)