Commit 278ce936 authored by Kenichi Handa's avatar Kenichi Handa
Browse files

(utf-16-decode-ucs): Look up

utf-subst-table-for-decode.  Fix for the case that the looking up
succeeds.
(ccl-decode-mule-utf-16-le): Translate characters by
utf-translation-table-for-decode.
(ccl-decode-mule-utf-16-be): Likewise.
(ccl-encode-mule-utf-16-le): Look up utf-subst-table-for-encode
at first.  Translate characters by
utf-translation-table-for-encode.
(ccl-encode-mule-utf-16-be): Likewise.
(mule-utf-16-le, mule-utf-16-be): Add `dependency' property.
parent 121223a9
...@@ -68,48 +68,49 @@ ...@@ -68,48 +68,49 @@
;; Needed in macro expansion, so can't be let-bound. Zapped after use. ;; Needed in macro expansion, so can't be let-bound. Zapped after use.
(eval-and-compile (eval-and-compile
(defconst utf-16-decode-ucs (defconst utf-16-decode-ucs
;; We have the unicode in r1. Output is character codes in r0, r1, ;; We have the unicode in r1. Output is charset ID in r0, code point
;; and r2 if appropriate. ;; in r1.
`((lookup-integer utf-8-subst-table r0 r3) `((lookup-integer utf-subst-table-for-decode r1 r3)
(if r7 (r1 = r3)) ; got a translation (if r7 ; got a translation
(if (r1 < 128) ((r0 = r1) (r1 = r3))
(r0 = ,(charset-id 'ascii)) (if (r1 < 128)
(if (r1 < 160) (r0 = ,(charset-id 'ascii))
(r0 = ,(charset-id 'eight-bit-control)) (if (r1 < 160)
(if (r1 < 256) (r0 = ,(charset-id 'eight-bit-control))
((r0 = ,(charset-id 'latin-iso8859-1)) (if (r1 < 256)
(r1 -= 128)) ((r0 = ,(charset-id 'latin-iso8859-1))
(if (r1 < #x2500) (r1 -= 128))
((r0 = ,(charset-id 'mule-unicode-0100-24ff)) (if (r1 < #x2500)
(r1 -= #x100) ((r0 = ,(charset-id 'mule-unicode-0100-24ff))
(r2 = (((r1 / 96) + 32) << 7)) (r1 -= #x100)
(r1 %= 96) (r2 = (((r1 / 96) + 32) << 7))
(r1 += (r2 + 32))) (r1 %= 96)
(if (r1 < #x3400) (r1 += (r2 + 32)))
((r0 = ,(charset-id 'mule-unicode-2500-33ff)) (if (r1 < #x3400)
(r1 -= #x2500) ((r0 = ,(charset-id 'mule-unicode-2500-33ff))
(r2 = (((r1 / 96) + 32) << 7)) (r1 -= #x2500)
(r1 %= 96) (r2 = (((r1 / 96) + 32) << 7))
(r1 += (r2 + 32))) (r1 %= 96)
(if (r1 < #xd800) ; 2 untranslated bytes (r1 += (r2 + 32)))
;; ;; Assume this is rare, so don't worry about the (if (r1 < #xd800) ; 2 untranslated bytes
;; ;; overhead of the call. ;; ;; Assume this is rare, so don't worry about the
;; (call mule-utf-16-untrans) ;; ;; overhead of the call.
((r0 = ,(charset-id 'mule-unicode-e000-ffff)) ;; (call mule-utf-16-untrans)
(r1 = 15037)) ; U+fffd ((r0 = ,(charset-id 'mule-unicode-e000-ffff))
(if (r1 < #xe000) ; surrogate (r1 = 15037)) ; U+fffd
;; ((call mule-utf-16-untrans) (if (r1 < #xe000) ; surrogate
;; (write-multibyte-character r0 r1) ;; ((call mule-utf-16-untrans)
;; (read r3 r4) ;; (write-multibyte-character r0 r1)
;; (call mule-utf-16-untrans)) ;; (read r3 r4)
((read r3 r4) ;; (call mule-utf-16-untrans))
(r0 = ,(charset-id 'mule-unicode-e000-ffff)) ((read r3 r4)
(r1 = 15037)) (r0 = ,(charset-id 'mule-unicode-e000-ffff))
((r0 = ,(charset-id 'mule-unicode-e000-ffff)) (r1 = 15037))
(r1 -= #xe000) ((r0 = ,(charset-id 'mule-unicode-e000-ffff))
(r2 = (((r1 / 96) + 32) << 7)) (r1 -= #xe000)
(r1 %= 96) (r2 = (((r1 / 96) + 32) << 7))
(r1 += (r2 + 32))))))))))))) (r1 %= 96)
(r1 += (r2 + 32))))))))))))))
(define-ccl-program ccl-decode-mule-utf-16-le (define-ccl-program ccl-decode-mule-utf-16-le
`(2 ; 2 bytes -> 1 to 4 bytes `(2 ; 2 bytes -> 1 to 4 bytes
...@@ -118,14 +119,14 @@ ...@@ -118,14 +119,14 @@
(read r3 r4) (read r3 r4)
(r1 = (r4 <8 r3)) (r1 = (r4 <8 r3))
,utf-16-decode-ucs ,utf-16-decode-ucs
(translate-character utf-8-translation-table-for-decode r0 r1) (translate-character utf-translation-table-for-decode r0 r1)
(write-multibyte-character r0 r1) (write-multibyte-character r0 r1)
(repeat)))) (repeat))))
"Decode little endian UTF-16 (ignoring signature bytes). "Decode little endian UTF-16 (ignoring signature bytes).
Basic decoding is done into the charsets ascii, latin-iso8859-1 and Basic decoding is done into the charsets ascii, latin-iso8859-1 and
mule-unicode-*. Un-representable Unicode characters are mule-unicode-*. Un-representable Unicode characters are decoded as
decoded as U+fffd. The result is run through translation table U+fffd. The result is run through the translation-table named
`utf-8-translation-table-for-decode' if that is defined.") `utf-translation-table-for-decode'.")
(define-ccl-program ccl-decode-mule-utf-16-be (define-ccl-program ccl-decode-mule-utf-16-be
`(2 ; 2 bytes -> 1 to 4 bytes `(2 ; 2 bytes -> 1 to 4 bytes
...@@ -134,14 +135,14 @@ decoded as U+fffd. The result is run through translation table ...@@ -134,14 +135,14 @@ decoded as U+fffd. The result is run through translation table
(read r3 r4) (read r3 r4)
(r1 = (r3 <8 r4)) (r1 = (r3 <8 r4))
,utf-16-decode-ucs ,utf-16-decode-ucs
(translate-character utf-8-translation-table-for-decode r0 r1) (translate-character utf-translation-table-for-decode r0 r1)
(write-multibyte-character r0 r1) (write-multibyte-character r0 r1)
(repeat)))) (repeat))))
"Decode big endian UTF-16 (ignoring signature bytes). "Decode big endian UTF-16 (ignoring signature bytes).
Basic decoding is done into the charsets ascii, latin-iso8859-1 and Basic decoding is done into the charsets ascii, latin-iso8859-1 and
mule-unicode-*. Un-representable Unicode characters are mule-unicode-*. Un-representable Unicode characters are
decoded as U+fffd. The result is run through translation table decoded as U+fffd. The result is run through the translation-table of
`utf-8-non-latin-8859-table'.") name `utf-translation-table-for-decode'.")
(makunbound 'utf-16-decode-ucs) ; done with it (makunbound 'utf-16-decode-ucs) ; done with it
...@@ -176,15 +177,18 @@ decoded as U+fffd. The result is run through translation table ...@@ -176,15 +177,18 @@ decoded as U+fffd. The result is run through translation table
(write #xfe) (write #xfe)
(loop (loop
(read-multibyte-character r0 r1) (read-multibyte-character r0 r1)
(translate-character ucs-mule-to-mule-unicode r0 r1) (lookup-character utf-subst-table-for-encode r0 r1)
,utf-16-decode-to-ucs (if (r7 == 0)
((translate-character utf-translation-table-for-encode r0 r1)
,utf-16-decode-to-ucs))
(write (r0 & 255)) (write (r0 & 255))
(write (r0 >> 8)) (write (r0 >> 8))
(repeat)))) (repeat))))
"Encode to little endian UTF-16 with signature. "Encode to little endian UTF-16 with signature.
Characters from the charsets ascii, eight-bit-control, Characters from the charsets ascii, eight-bit-control,
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
after translation through the table `ucs-mule-to-mule-unicode'. after translation through the translation-table of name
`utf-translation-table-for-encode'.
Others are encoded as U+FFFD.") Others are encoded as U+FFFD.")
(define-ccl-program ccl-encode-mule-utf-16-be (define-ccl-program ccl-encode-mule-utf-16-be
...@@ -193,15 +197,18 @@ Others are encoded as U+FFFD.") ...@@ -193,15 +197,18 @@ Others are encoded as U+FFFD.")
(write #xff) (write #xff)
(loop (loop
(read-multibyte-character r0 r1) (read-multibyte-character r0 r1)
(translate-character ucs-mule-to-mule-unicode r0 r1) (lookup-character utf-subst-table-for-encode r0 r1)
,utf-16-decode-to-ucs (if (r7 == 0)
((translate-character utf-translation-table-for-encode r0 r1)
,utf-16-decode-to-ucs))
(write (r0 >> 8)) (write (r0 >> 8))
(write (r0 & 255)) (write (r0 & 255))
(repeat)))) (repeat))))
"Encode to big endian UTF-16 with signature. "Encode to big endian UTF-16 with signature.
Characters from the charsets ascii, eight-bit-control, Characters from the charsets ascii, eight-bit-control,
eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded eight-bit-graphic, latin-iso8859-1 and mule-unicode-* are encoded
after translation through the table `ucs-mule-to-mule-unicode'. after translation through the translation-table named
`utf-translation-table-for-encode'.
Others are encoded as U+FFFD.") Others are encoded as U+FFFD.")
(makunbound 'utf-16-decode-to-ucs) (makunbound 'utf-16-decode-to-ucs)
...@@ -210,20 +217,19 @@ Others are encoded as U+FFFD.") ...@@ -210,20 +217,19 @@ Others are encoded as U+FFFD.")
Assumes and ignores the leading two-byte signature. Assumes and ignores the leading two-byte signature.
The supported Emacs character sets are the following, plus others It supports Unicode characters of these ranges:
which may be included in the translation table U+0000..U+33FF, U+E000..U+FFFF.
`ucs-mule-to-mule-unicode': They correspond to these Emacs character sets:
ascii ascii, latin-iso8859-1, mule-unicode-0100-24ff,
eight-bit-control mule-unicode-2500-33ff, mule-unicode-e000-ffff
latin-iso8859-1
mule-unicode-0100-24ff On decoding (e.g. reading a file), Unicode characters not in the above
mule-unicode-2500-33ff ranges are decoded as U+FFFD, effectively corrupting the data
mule-unicode-e000-ffff if they are re-encoded.
Note that Unicode characters out of the ranges U+0000-U+33FF and On encoding (e.g. writing a file), Emacs characters not belonging to
U+E200-U+FFFF are decoded as U+FFFD, effectively corrupting the data any of the character sets listed above are encoded into the byte
if they are re-encoded. Emacs characters without Unicode conversions sequence representing U+FFFD (REPLACEMENT CHARACTER)."))
are encoded as U+FFFD."))
(make-coding-system (make-coding-system
'mule-utf-16-le 4 'mule-utf-16-le 4
?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u. ?u ; Mule-UCS uses ?U, but code-pages uses that for koi8-u.
...@@ -242,7 +248,11 @@ are encoded as U+FFFD.")) ...@@ -242,7 +248,11 @@ are encoded as U+FFFD."))
(mime-charset . utf-16le) (mime-charset . utf-16le)
(coding-category . coding-category-utf-16-le) (coding-category . coding-category-utf-16-le)
(valid-codes (0 . 255)) (valid-codes (0 . 255))
(pre-write-conversion . utf-16-le-pre-write-conversion))) (pre-write-conversion . utf-16-le-pre-write-conversion)
(dependency unify-8859-on-encoding-mode
unify-8859-on-decoding-mode
utf-fragment-on-decoding
utf-translate-cjk)))
(make-coding-system (make-coding-system
'mule-utf-16-be 4 ?u 'mule-utf-16-be 4 ?u
...@@ -261,10 +271,11 @@ are encoded as U+FFFD.")) ...@@ -261,10 +271,11 @@ are encoded as U+FFFD."))
(mime-charset . utf-16be) (mime-charset . utf-16be)
(coding-category . coding-category-utf-16-be) (coding-category . coding-category-utf-16-be)
(valid-codes (0 . 255)) (valid-codes (0 . 255))
(pre-write-conversion . utf-16-be-pre-write-conversion))) (pre-write-conversion . utf-16-be-pre-write-conversion)
(dependency unify-8859-on-encoding-mode
(register-char-codings 'mule-utf-16-le ucs-mule-to-mule-unicode) unify-8859-on-decoding-mode
(register-char-codings 'mule-utf-16-be ucs-mule-to-mule-unicode)) utf-fragment-on-decoding
utf-translate-cjk))))
(define-coding-system-alias 'utf-16-le 'mule-utf-16-le) (define-coding-system-alias 'utf-16-le 'mule-utf-16-le)
(define-coding-system-alias 'utf-16-be 'mule-utf-16-be) (define-coding-system-alias 'utf-16-be 'mule-utf-16-be)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment