mapconv 4.79 KB
Newer Older
Kenichi Handa's avatar
Kenichi Handa committed
1
#!/bin/sh
2

Paul Eggert's avatar
Paul Eggert committed
3
# Copyright (C) 2015-2017 Free Software Foundation, Inc.
4

5
# Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
Kenichi Handa's avatar
Kenichi Handa committed
6 7
#   National Institute of Advanced Industrial Science and Technology (AIST)
#   Registration Number H13PRO009
8

Kenichi Handa's avatar
Kenichi Handa committed
9
# This file is part of GNU Emacs.
10 11

# GNU Emacs is free software: you can redistribute it and/or modify
Kenichi Handa's avatar
Kenichi Handa committed
12
# it under the terms of the GNU General Public License as published by
13 14 15
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

Kenichi Handa's avatar
Kenichi Handa committed
16 17 18 19
# GNU Emacs is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
20

Kenichi Handa's avatar
Kenichi Handa committed
21
# You should have received a copy of the GNU General Public License
22 23 24
# along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.

# Commentary:
Kenichi Handa's avatar
Kenichi Handa committed
25 26 27 28 29 30 31 32 33 34

# Convert charset map of various format into this:
#	0xXX 0xYYYY
# where,
#   XX is a code point of the charset in hexa-decimal,
#   YYYY is the corresponding Unicode character code in hexa-decimal.
# Arguments are:
#   $1: source map file
#   $2: address pattern for sed (optionally with substitution command)
#   $3: format of source map file
35
#	GLIBC-1 GLIBC-2 GLIBC-2-7 CZYBORRA IANA UNICODE UNICODE2 YASUOKA
Kenichi Handa's avatar
Kenichi Handa committed
36 37
#   $4: awk script

38 39 40 41 42
## So that eg [A-F] as used by KANJI-DATABASE branch below works as expected.
## Otherwise with LANG=en_US.utf8, CNS-6.map was generated with a
## bogus entry.  By experiment, LC_COLLATE=C was not enough.
export LC_ALL=C

43 44 45 46
BASE=`expr "$1" : '.*/\(.*\)' '|' "$1"` # basename
FILE="admin/charsets/mapfiles/$BASE"
BASE=`expr "$BASE" : '\(.*\)\.gz$' '|' "$BASE"` # remove any .gz suffix
AWK=${AWK:-awk}
Kenichi Handa's avatar
Kenichi Handa committed
47 48 49

case "$3" in
    GLIBC*)
50
	FILE="$BASE in localedata/charmaps of glibc";
51
	SOURCE="";;
Kenichi Handa's avatar
Kenichi Handa committed
52
    CZYBORRA)
53
	BASE="$BASE.gz";
Kenichi Handa's avatar
Kenichi Handa committed
54 55 56 57
	SOURCE="http://czyborra.com/charsets/${BASE}";;
    IANA)
	SOURCE="http://www.iana.org/assignments/charset-reg/${BASE}";;
    UNICODE)
58
	SOURCE="http://www.unicode.org/Public/MAPPINGS/VENDORS/ADOBE/${BASE}";;
Kenichi Handa's avatar
Kenichi Handa committed
59
    UNICODE2)
60
	SOURCE="http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/${BASE}";;
Kenichi Handa's avatar
Kenichi Handa committed
61
    YASUOKA)
62 63
	BASE="$BASE.Z";
	SOURCE="http://kanji.zinbun.kyoto-u.ac.jp/~yasuoka/ftp/CJKtable/${BASE}";;
64
    KANJI-DATABASE)
65
	SOURCE="http://kanji-database.cvs.sourceforge.net/viewvc/*checkout*/kanji-database/kanji-database/data/cns2ucsdkw.txt?revision=1.4";;
Kenichi Handa's avatar
Kenichi Handa committed
66
    *)
67
	printf 'Unknown file type: %s\n' "$3"
Kenichi Handa's avatar
Kenichi Handa committed
68 69 70
	exit 1;;
esac

71 72 73 74 75 76 77
if [ -n "$SOURCE" ] ; then
    echo "# Generated from $FILE which is a copy of";
    echo "# $SOURCE"
else
    echo "# Generated from $FILE"
fi

Kenichi Handa's avatar
Kenichi Handa committed
78 79 80

if [ -n "$4" ] ; then
    if [ -f "$4" ] ; then
81
	AWKPROG="$AWK -f $4"
Kenichi Handa's avatar
Kenichi Handa committed
82 83 84 85 86 87 88 89
    else
	echo "Awk program does not exist: $4"
	exit 1
    fi
else
    AWKPROG=cat
fi

90
if [ "$3" = "GLIBC-1" ] ; then
Kenichi Handa's avatar
Kenichi Handa committed
91 92
    # Source format is:
    #   <UYYYY>	/xXX
93
    gunzip -c $1 | sed -n -e "${2}p" \
Kenichi Handa's avatar
Kenichi Handa committed
94 95
	| sed -e 's,<U\([^>]*\)>[ 	]*/x\(..\).*,0x\2 0x\1,' \
	| sort | ${AWKPROG}
96
elif [ "$3" = "GLIBC-2" ] ; then
Kenichi Handa's avatar
Kenichi Handa committed
97 98
    # Source format is:
    #   <UYYYY>	/xXX/xZZ
99
    gunzip -c $1 | sed -n -e "${2}p" \
Kenichi Handa's avatar
Kenichi Handa committed
100 101
	| sed -e 's,<U\([^>]*\)>[ 	]*/x\(..\)/x\(..\).*,0x\2\3 0x\1,' \
	| sort | ${AWKPROG}
102
elif [ "$3" = "GLIBC-2-7" ] ; then
Kenichi Handa's avatar
Kenichi Handa committed
103 104 105
    # Source format is:
    #   <UYYYY>	/xXX/xZZ
    # We must drop MSBs of XX and ZZ
106
    gunzip -c $1 | sed -n -e "${2}p" \
Kenichi Handa's avatar
Kenichi Handa committed
107 108 109 110
	| sed -e 's/xa/x2/g' -e 's/xb/x3/g' -e 's/xc/x4/g' \
	      -e 's/xd/x5/g' -e 's/xe/x6/g' -e 's/xf/x7/g' \
	      -e 's,<U\([^>]*\)>[ 	]*/x\(..\)/x\(..\).*,0x\2\3 0x\1,' \
	| sort | ${AWKPROG}
111
elif [ "$3" = "CZYBORRA" ] ; then
Kenichi Handa's avatar
Kenichi Handa committed
112 113
    # Source format is:
    #   =XX	U+YYYY
114
    sed -n -e "${2}p" < $1 \
Kenichi Handa's avatar
Kenichi Handa committed
115 116
	| sed -e 's/=\(..\)[^U]*U+\([0-9A-F]*\).*/0x\1 0x\2/' \
	| sort | ${AWKPROG}
117
elif [ "$3" = "IANA" ] ; then
Kenichi Handa's avatar
Kenichi Handa committed
118 119
    # Source format is:
    #   0xXX	0xYYYY
120
    sed -n -e "${2}p" < $1 \
Kenichi Handa's avatar
Kenichi Handa committed
121 122
	| sed -e 's/\(0x[0-9A-Fa-f]*\)[^0]*\(0x[0-9A-Fa-f]*\).*/\1 \2/' \
	| sort | ${AWKPROG}
123
elif [ "$3" = "UNICODE" ] ; then
Kenichi Handa's avatar
Kenichi Handa committed
124 125
    # Source format is:
    #   YYYY	XX
126 127
    # We perform reverse sort to prefer the first one in the
    # duplicated mappings (e.g. 0x20->U+0020, 0x20->U+00A0).
128
    sed -n -e "${2}p" < $1 \
Kenichi Handa's avatar
Kenichi Handa committed
129
	| sed -e 's/\([0-9A-F]*\)[^0-9A-F]*\([0-9A-F]*\).*/0x\2 0x\1/' \
130
	| sort -r
131
elif [ "$3" = "UNICODE2" ] ; then
Kenichi Handa's avatar
Kenichi Handa committed
132 133
    # Source format is:
    #   0xXXXX	0xYYYY	# ...
134
    sed -n -e "${2}p" < $1 \
Kenichi Handa's avatar
Kenichi Handa committed
135
	| sed -e 's/\([0-9A-Fx]*\)[^0]*\([0-9A-Fx]*\).*/\1 \2/' \
136
	| ${AWKPROG} | sort -n -k 4,4
137
elif [ "$3" = "YASUOKA" ] ; then
Kenichi Handa's avatar
Kenichi Handa committed
138 139
    # Source format is:
    # YYYY	0-XXXX (XXXX is a Kuten code)
140
    sed -n -e "${2}p" < $1 \
Kenichi Handa's avatar
Kenichi Handa committed
141 142
	| sed -e 's/\([0-9A-F]*\)[^0]*0-\([0-9]*\).*/0x\2 0x\1/' \
	| sort | ${AWKPROG}
143
elif [ "$3" = "KANJI-DATABASE" ] ; then
144 145
    # Source format is:
    # C?-XXXX U+YYYYY .....
146
    sed -n -e "${2}p" < $1 \
147 148
	| sed -e 's/...\(....\) U+\([0-9A-F]*\).*/0x\1 0x\2/' \
	| sort | ${AWKPROG}
Kenichi Handa's avatar
Kenichi Handa committed
149
else
150
    printf 'Invalid arguments: %s\n' "$3"
Kenichi Handa's avatar
Kenichi Handa committed
151 152
    exit 1
fi