|
502
|
1 ;;; mule-coding.el --- Coding-system functions for Mule. -*- coding: iso-2022-7bit; -*-
|
|
333
|
2
|
|
|
3 ;; Copyright (C) 1995 Electrotechnical Laboratory, JAPAN.
|
|
|
4 ;; Licensed to the Free Software Foundation.
|
|
|
5 ;; Copyright (C) 1995 Amdahl Corporation.
|
|
|
6 ;; Copyright (C) 1995 Sun Microsystems.
|
|
|
7 ;; Copyright (C) 1997 MORIOKA Tomohiko
|
|
771
|
8 ;; Copyright (C) 2001 Ben Wing.
|
|
333
|
9
|
|
|
10 ;; This file is part of XEmacs.
|
|
|
11
|
|
|
12 ;; XEmacs is free software; you can redistribute it and/or modify it
|
|
|
13 ;; under the terms of the GNU General Public License as published by
|
|
|
14 ;; the Free Software Foundation; either version 2, or (at your option)
|
|
|
15 ;; any later version.
|
|
|
16
|
|
|
17 ;; XEmacs is distributed in the hope that it will be useful, but
|
|
|
18 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
20 ;; General Public License for more details.
|
|
|
21
|
|
|
22 ;; You should have received a copy of the GNU General Public License
|
|
444
|
23 ;; along with XEmacs; see the file COPYING. If not, write to the
|
|
333
|
24 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
|
25 ;; Boston, MA 02111-1307, USA.
|
|
|
26
|
|
|
27 ;;; Commentary:
|
|
|
28
|
|
|
29 ;;; split off of mule.el and mostly moved to coding.el
|
|
|
30
|
|
4072
|
31 ;; Needed for make-8-bit-coding-system.
|
|
4080
|
32 (eval-when-compile (require 'ccl))
|
|
4072
|
33
|
|
333
|
34 ;;; Code:
|
|
|
35
|
|
|
36 (defun coding-system-force-on-output (coding-system register)
|
|
|
37 "Return the 'force-on-output property of CODING-SYSTEM for the specified REGISTER."
|
|
444
|
38 (check-type register integer)
|
|
333
|
39 (coding-system-property
|
|
|
40 coding-system
|
|
|
41 (case register
|
|
|
42 (0 'force-g0-on-output)
|
|
|
43 (1 'force-g1-on-output)
|
|
|
44 (2 'force-g2-on-output)
|
|
|
45 (3 'force-g3-on-output)
|
|
|
46 (t (signal 'args-out-of-range (list register 0 3))))))
|
|
|
47
|
|
|
48 (defun coding-system-short (coding-system)
|
|
|
49 "Return the 'short property of CODING-SYSTEM."
|
|
|
50 (coding-system-property coding-system 'short))
|
|
|
51
|
|
|
52 (defun coding-system-no-ascii-eol (coding-system)
|
|
|
53 "Return the 'no-ascii-eol property of CODING-SYSTEM."
|
|
|
54 (coding-system-property coding-system 'no-ascii-eol))
|
|
|
55
|
|
|
56 (defun coding-system-no-ascii-cntl (coding-system)
|
|
|
57 "Return the 'no-ascii-cntl property of CODING-SYSTEM."
|
|
|
58 (coding-system-property coding-system 'no-ascii-cntl))
|
|
|
59
|
|
|
60 (defun coding-system-seven (coding-system)
|
|
|
61 "Return the 'seven property of CODING-SYSTEM."
|
|
|
62 (coding-system-property coding-system 'seven))
|
|
|
63
|
|
|
64 (defun coding-system-lock-shift (coding-system)
|
|
|
65 "Return the 'lock-shift property of CODING-SYSTEM."
|
|
|
66 (coding-system-property coding-system 'lock-shift))
|
|
|
67
|
|
|
68 ;;(defun coding-system-use-japanese-jisx0201-roman (coding-system)
|
|
|
69 ;; "Return the 'use-japanese-jisx0201-roman property of CODING-SYSTEM."
|
|
|
70 ;; (coding-system-property coding-system 'use-japanese-jisx0201-roman))
|
|
|
71
|
|
|
72 ;;(defun coding-system-use-japanese-jisx0208-1978 (coding-system)
|
|
|
73 ;; "Return the 'use-japanese-jisx0208-1978 property of CODING-SYSTEM."
|
|
|
74 ;; (coding-system-property coding-system 'use-japanese-jisx0208-2978))
|
|
|
75
|
|
|
76 (defun coding-system-no-iso6429 (coding-system)
|
|
|
77 "Return the 'no-iso6429 property of CODING-SYSTEM."
|
|
|
78 (coding-system-property coding-system 'no-iso6429))
|
|
|
79
|
|
|
80 (defun coding-system-ccl-encode (coding-system)
|
|
|
81 "Return the CCL 'encode property of CODING-SYSTEM."
|
|
|
82 (coding-system-property coding-system 'encode))
|
|
|
83
|
|
|
84 (defun coding-system-ccl-decode (coding-system)
|
|
|
85 "Return the CCL 'decode property of CODING-SYSTEM."
|
|
|
86 (coding-system-property coding-system 'decode))
|
|
|
87
|
|
771
|
88 (defun coding-system-iso2022-charset (coding-system register)
|
|
|
89 "Return the charset initially designated to REGISTER in CODING-SYSTEM.
|
|
|
90 The allowable range of REGISTER is 0 through 3."
|
|
|
91 (if (or (< register 0) (> register 3))
|
|
|
92 (error 'args-out-of-range "coding-system-charset REGISTER" register 0 3))
|
|
|
93 (coding-system-property coding-system (nth register '(charset-g0
|
|
|
94 charset-g1
|
|
|
95 charset-g2
|
|
|
96 charset-g3))))
|
|
|
97
|
|
333
|
98
|
|
|
99 ;;;; Definitions of predefined coding systems
|
|
|
100
|
|
|
101 (make-coding-system
|
|
|
102 'ctext 'iso2022
|
|
771
|
103 "Compound Text"
|
|
333
|
104 '(charset-g0 ascii
|
|
|
105 charset-g1 latin-iso8859-1
|
|
|
106 eol-type nil
|
|
|
107 mnemonic "CText"))
|
|
|
108
|
|
|
109 (make-coding-system
|
|
|
110 'iso-8859-1 'no-conversion
|
|
771
|
111 "ISO-8859-1 (Latin-1)"
|
|
333
|
112 '(eol-type nil mnemonic "Noconv"))
|
|
|
113
|
|
|
114 (make-coding-system
|
|
|
115 'iso-2022-8bit-ss2 'iso2022
|
|
771
|
116 "ISO-2022 8-bit w/SS2"
|
|
333
|
117 '(charset-g0 ascii
|
|
|
118 charset-g1 latin-iso8859-1
|
|
|
119 charset-g2 t ;; unspecified but can be used later.
|
|
|
120 short t
|
|
|
121 mnemonic "ISO8/SS"
|
|
771
|
122 documentation "ISO 2022 based 8-bit encoding using SS2 for 96-charset"
|
|
333
|
123 ))
|
|
|
124
|
|
|
125 (make-coding-system
|
|
|
126 'iso-2022-7bit-ss2 'iso2022
|
|
771
|
127 "ISO-2022 7-bit w/SS2"
|
|
333
|
128 '(charset-g0 ascii
|
|
|
129 charset-g2 t ;; unspecified but can be used later.
|
|
|
130 seven t
|
|
|
131 short t
|
|
|
132 mnemonic "ISO7/SS"
|
|
771
|
133 documentation "ISO 2022 based 7-bit encoding using SS2 for 96-charset"
|
|
333
|
134 eol-type nil))
|
|
|
135
|
|
|
136 ;; (copy-coding-system 'iso-2022-7bit-ss2 'iso-2022-jp-2)
|
|
|
137 (make-coding-system
|
|
|
138 'iso-2022-jp-2 'iso2022
|
|
771
|
139 "ISO-2022-JP-2"
|
|
333
|
140 '(charset-g0 ascii
|
|
|
141 charset-g2 t ;; unspecified but can be used later.
|
|
|
142 seven t
|
|
|
143 short t
|
|
|
144 mnemonic "ISO7/SS"
|
|
|
145 eol-type nil))
|
|
|
146
|
|
|
147 (make-coding-system
|
|
|
148 'iso-2022-7bit 'iso2022
|
|
771
|
149 "ISO 2022 7-bit"
|
|
333
|
150 '(charset-g0 ascii
|
|
|
151 seven t
|
|
|
152 short t
|
|
771
|
153 mnemonic "ISO7"
|
|
|
154 documentation "ISO-2022-based 7-bit encoding using only G0"
|
|
|
155 ))
|
|
333
|
156
|
|
|
157 ;; compatibility for old XEmacsen
|
|
771
|
158 (define-coding-system-alias 'iso-2022-7 'iso-2022-7bit)
|
|
333
|
159
|
|
|
160 (make-coding-system
|
|
|
161 'iso-2022-8 'iso2022
|
|
771
|
162 "ISO-2022 8-bit"
|
|
333
|
163 '(charset-g0 ascii
|
|
|
164 charset-g1 latin-iso8859-1
|
|
|
165 short t
|
|
|
166 mnemonic "ISO8"
|
|
771
|
167 documentation "ISO-2022 eight-bit coding system. No single-shift or locking-shift."
|
|
333
|
168 ))
|
|
|
169
|
|
|
170 (make-coding-system
|
|
|
171 'escape-quoted 'iso2022
|
|
771
|
172 "Escape-Quoted (for .ELC files)"
|
|
333
|
173 '(charset-g0 ascii
|
|
|
174 charset-g1 latin-iso8859-1
|
|
|
175 eol-type lf
|
|
|
176 escape-quoted t
|
|
|
177 mnemonic "ESC/Quot"
|
|
771
|
178 documentation "ISO-2022 eight-bit coding system with escape quoting; used for .ELC files."
|
|
333
|
179 ))
|
|
|
180
|
|
|
181 (make-coding-system
|
|
|
182 'iso-2022-lock 'iso2022
|
|
771
|
183 "ISO-2022 w/locking-shift"
|
|
333
|
184 '(charset-g0 ascii
|
|
|
185 charset-g1 t ;; unspecified but can be used later.
|
|
|
186 seven t
|
|
|
187 lock-shift t
|
|
|
188 mnemonic "ISO7/Lock"
|
|
771
|
189 documentation "ISO-2022 coding system using Locking-Shift for 96-charset."
|
|
333
|
190 ))
|
|
4072
|
191
|
|
333
|
192
|
|
4072
|
193 ;; This is used by people writing CCL programs, but is called at runtime.
|
|
|
194 (defun define-translation-hash-table (symbol table)
|
|
|
195 "Define SYMBOL as the name of the hash translation TABLE for use in CCL.
|
|
|
196
|
|
|
197 Analogous to `define-translation-table', but updates
|
|
|
198 `translation-hash-table-vector' and the table is for use in the CCL
|
|
|
199 `lookup-integer' and `lookup-character' functions."
|
|
|
200 (unless (and (symbolp symbol)
|
|
|
201 (hash-table-p table))
|
|
|
202 (error "Bad args to define-translation-hash-table"))
|
|
|
203 (let ((len (length translation-hash-table-vector))
|
|
|
204 (id 0)
|
|
|
205 done)
|
|
|
206 (put symbol 'translation-hash-table table)
|
|
|
207 (while (not done)
|
|
|
208 (if (>= id len)
|
|
|
209 (setq translation-hash-table-vector
|
|
|
210 (vconcat translation-hash-table-vector [nil])))
|
|
|
211 (let ((slot (aref translation-hash-table-vector id)))
|
|
|
212 (if (or (not slot)
|
|
|
213 (eq (car slot) symbol))
|
|
|
214 (progn
|
|
|
215 (aset translation-hash-table-vector id (cons symbol table))
|
|
|
216 (setq done t))
|
|
|
217 (setq id (1+ id)))))
|
|
|
218 (put symbol 'translation-hash-table-id id)
|
|
|
219 id))
|
|
|
220
|
|
|
221 (defvar make-8-bit-private-use-start (decode-char 'ucs #xE000)
|
|
|
222 "Start of a 256 code private use area for make-8-bit-coding-system.
|
|
|
223
|
|
|
224 This is used to ensure that distinct octets on disk for a given coding
|
|
|
225 system map to distinct XEmacs characters, preventing a spurious changes when
|
|
|
226 a file is read, not changed, and then written. ")
|
|
|
227
|
|
|
228 (defun make-8-bit-generate-helper (decode-table encode-table
|
|
|
229 encode-failure-octet)
|
|
|
230 "Helper function for `make-8-bit-generate-encode-program', which see.
|
|
|
231
|
|
|
232 Deals with the case where ASCII and another character set provide the
|
|
|
233 can both be encoded unambiguously into the coding-system; if this is
|
|
|
234 so, returns a list corresponding to such a ccl-program. If not, it
|
|
|
235 returns nil. "
|
|
|
236 (let ((tentative-encode-program-parts
|
|
|
237 (eval-when-compile
|
|
|
238 (let* ((compiled
|
|
|
239 (append
|
|
|
240 (ccl-compile
|
|
|
241 `(1
|
|
|
242 (loop
|
|
|
243 (read-multibyte-character r0 r1)
|
|
|
244 (if (r0 == ,(charset-id 'ascii))
|
|
|
245 (write r1)
|
|
|
246 ((if (r0 == #xABAB)
|
|
|
247 ;; #xBFFE is a sentinel in the compiled
|
|
|
248 ;; program.
|
|
|
249 (write r1 ,(make-vector 256 #xBFFE))
|
|
|
250 ((mule-to-unicode r0 r1)
|
|
|
251 (if (r0 == #xFFFD)
|
|
|
252 (write #xBEEF)
|
|
|
253 ((lookup-integer encode-table-sym r0 r3)
|
|
|
254 (if r7
|
|
|
255 (write-multibyte-character r0 r3)
|
|
|
256 (write #xBEEF))))))))
|
|
|
257 (repeat)))) nil))
|
|
|
258 (first-part compiled)
|
|
|
259 (last-part
|
|
|
260 (member-if-not (lambda (entr) (eq #xBFFE entr))
|
|
|
261 (member-if
|
|
|
262 (lambda (entr) (eq #xBFFE entr))
|
|
|
263 first-part))))
|
|
|
264 (while compiled
|
|
|
265 (if (eq #xBFFE (cadr compiled))
|
|
|
266 (setcdr compiled nil))
|
|
|
267 (setq compiled (cdr compiled)))
|
|
|
268 ;; Is the generated code as we expect it to be?
|
|
|
269 (assert (and (memq #xABAB first-part)
|
|
|
270 (memq #xBEEF14 last-part))
|
|
|
271 nil
|
|
|
272 "This code assumes that the constant #xBEEF is #xBEEF14 in \
|
|
|
273 compiled CCL code,\nand that the constant #xABAB is #xABAB. If that is
|
|
|
274 not the case, and it appears not to be--that's why you're getting this
|
|
|
275 message--it will not work. ")
|
|
|
276 (list first-part last-part))))
|
|
|
277 (charset-lower -1)
|
|
|
278 (charset-upper -1)
|
|
|
279 worth-trying known-charsets encode-program
|
|
|
280 other-charset-vector ucs)
|
|
|
281
|
|
|
282 (loop for char across decode-table
|
|
|
283 do (pushnew (char-charset char) known-charsets))
|
|
|
284 (setq known-charsets (delq 'ascii known-charsets))
|
|
|
285
|
|
|
286 (loop for known-charset in known-charsets
|
|
|
287 do
|
|
|
288 ;; This is not possible for two dimensional charsets.
|
|
|
289 (when (eq 1 (charset-dimension known-charset))
|
|
|
290 (setq args-out-of-range t)
|
|
|
291 (if (eq 'control-1 known-charset)
|
|
|
292 (setq charset-lower 0
|
|
|
293 charset-upper 31)
|
|
|
294 ;; There should be a nicer way to get the limits here.
|
|
|
295 (condition-case args-out-of-range
|
|
|
296 (make-char known-charset #x100)
|
|
|
297 (args-out-of-range
|
|
|
298 (setq charset-lower (third args-out-of-range)
|
|
|
299 charset-upper (fourth args-out-of-range)))))
|
|
|
300 (loop
|
|
|
301 for i from charset-lower to charset-upper
|
|
|
302 always (and (setq ucs
|
|
|
303 (encode-char (make-char known-charset i) 'ucs))
|
|
|
304 (gethash ucs encode-table))
|
|
|
305 finally (setq worth-trying known-charset))
|
|
|
306
|
|
|
307 ;; Only trying this for one charset at a time, the first find.
|
|
|
308 (when worth-trying (return))
|
|
|
309
|
|
|
310 ;; Okay, this charset is not worth trying, Try the next.
|
|
|
311 (setq charset-lower -1
|
|
|
312 charset-upper -1
|
|
|
313 worth-trying nil)))
|
|
|
314
|
|
|
315 (when worth-trying
|
|
|
316 (setq other-charset-vector (make-vector 256 encode-failure-octet))
|
|
|
317 (loop for i from charset-lower to charset-upper
|
|
|
318 do (aset other-charset-vector (+ #x80 i)
|
|
|
319 (gethash (encode-char (make-char worth-trying i)
|
|
|
320 'ucs) encode-table)))
|
|
|
321 (setq encode-program
|
|
|
322 (nsublis
|
|
|
323 (list (cons #xABAB (charset-id worth-trying)))
|
|
|
324 (nconc
|
|
|
325 (copy-list (first
|
|
|
326 tentative-encode-program-parts))
|
|
|
327 (append other-charset-vector nil)
|
|
|
328 (copy-tree (second
|
|
|
329 tentative-encode-program-parts))))))
|
|
|
330 encode-program))
|
|
|
331
|
|
|
332 (defun make-8-bit-generate-encode-program (decode-table encode-table
|
|
|
333 encode-failure-octet)
|
|
|
334 "Generate a CCL program to decode a 8-bit fixed-width charset.
|
|
|
335
|
|
|
336 DECODE-TABLE must have 256 non-cons entries, and will be regarded as
|
|
|
337 describing a map from the octet corresponding to an offset in the
|
|
|
338 table to the that entry in the table. ENCODE-TABLE is a hash table
|
|
|
339 map from unicode values to characters in the range [0,255].
|
|
|
340 ENCODE-FAILURE-OCTET describes an integer between 0 and 255
|
|
|
341 \(inclusive) to write in the event that a character cannot be encoded. "
|
|
|
342 (check-argument-type #'vectorp decode-table)
|
|
|
343 (check-argument-range (length decode-table) #x100 #x100)
|
|
|
344 (check-argument-type #'hash-table-p encode-table)
|
|
|
345 (check-argument-type #'integerp encode-failure-octet)
|
|
|
346 (check-argument-range encode-failure-octet #x00 #xFF)
|
|
|
347 (let ((encode-program nil)
|
|
|
348 (general-encode-program
|
|
|
349 (eval-when-compile
|
|
|
350 (let ((prog (append
|
|
|
351 (ccl-compile
|
|
|
352 `(1
|
|
|
353 (loop
|
|
|
354 (read-multibyte-character r0 r1)
|
|
|
355 (mule-to-unicode r0 r1)
|
|
|
356 (if (r0 == #xFFFD)
|
|
|
357 (write #xBEEF)
|
|
|
358 ((lookup-integer encode-table-sym r0 r3)
|
|
|
359 (if r7
|
|
|
360 (write-multibyte-character r0 r3)
|
|
|
361 (write #xBEEF))))
|
|
|
362 (repeat)))) nil)))
|
|
|
363 (assert (memq #xBEEF14 prog)
|
|
|
364 nil
|
|
|
365 "This code assumes that the constant #xBEEF is #xBEEF14 \
|
|
|
366 in compiled CCL code.\nIf that is not the case, and it appears not to
|
|
|
367 be--that's why you're getting this message--it will not work. ")
|
|
|
368 prog)))
|
|
|
369 (encode-program-with-ascii-optimisation
|
|
|
370 (eval-when-compile
|
|
|
371 (let ((prog (append
|
|
|
372 (ccl-compile
|
|
|
373 `(1
|
|
|
374 (loop
|
|
|
375 (read-multibyte-character r0 r1)
|
|
|
376 (if (r0 == ,(charset-id 'ascii))
|
|
|
377 (write r1)
|
|
|
378 ((mule-to-unicode r0 r1)
|
|
|
379 (if (r0 == #xFFFD)
|
|
|
380 (write #xBEEF)
|
|
|
381 ((lookup-integer encode-table-sym r0 r3)
|
|
|
382 (if r7
|
|
|
383 (write-multibyte-character r0 r3)
|
|
|
384 (write #xBEEF))))))
|
|
|
385 (repeat)))) nil)))
|
|
|
386 (assert (memq #xBEEF14 prog)
|
|
|
387 nil
|
|
|
388 "This code assumes that the constant #xBEEF is #xBEEF14 \
|
|
|
389 in compiled CCL code.\nIf that is not the case, and it appears not to
|
|
|
390 be--that's why you're getting this message--it will not work. ")
|
|
|
391 prog)))
|
|
|
392 (ascii-encodes-as-itself nil))
|
|
|
393
|
|
|
394 ;; Is this coding system ASCII-compatible? If so, we can avoid the hash
|
|
|
395 ;; table lookup for those characters.
|
|
|
396 (loop
|
|
|
397 for i from #x00 to #x7f
|
|
|
398 always (eq (int-to-char i) (gethash i encode-table))
|
|
|
399 finally (setq ascii-encodes-as-itself t))
|
|
|
400
|
|
|
401 ;; Note that this logic handles EBCDIC badly. For example, CP037,
|
|
|
402 ;; MIME name ebcdic-na, has the entire repertoire of ASCII and
|
|
|
403 ;; Latin 1, and thus a more optimal ccl encode program would check
|
|
|
404 ;; for those character sets and use tables. But for now, we do a
|
|
|
405 ;; hash table lookup for every character.
|
|
|
406 (if (null ascii-encodes-as-itself)
|
|
|
407 ;; General encode program. Pros; general and correct. Cons;
|
|
|
408 ;; slow, a hash table lookup + mule-unicode conversion is done
|
|
|
409 ;; for every character encoding.
|
|
|
410 (setq encode-program general-encode-program)
|
|
|
411 (setq encode-program
|
|
|
412 ;; Encode program with ascii-ascii mapping (based on a
|
|
|
413 ;; character's mule character set), and one other mule
|
|
|
414 ;; character set using table-based encoding, other
|
|
|
415 ;; character sets using hash table lookups.
|
|
|
416 ;; make-8-bit-non-ascii-completely-coveredp only returns
|
|
|
417 ;; such a mapping if some non-ASCII charset with
|
|
|
418 ;; characters in decode-table is entirely covered by
|
|
|
419 ;; encode-table.
|
|
|
420 (make-8-bit-generate-helper decode-table encode-table
|
|
|
421 encode-failure-octet))
|
|
|
422 (unless encode-program
|
|
|
423 ;; If make-8-bit-non-ascii-completely-coveredp returned nil,
|
|
|
424 ;; but ASCII still encodes as itself, do one-to-one mapping
|
|
|
425 ;; for ASCII, and a hash table lookup for everything else.
|
|
|
426 (setq encode-program encode-program-with-ascii-optimisation)))
|
|
|
427
|
|
|
428 (setq encode-program
|
|
|
429 (nsublis
|
|
|
430 (list (cons #xBEEF14
|
|
|
431 (logior (lsh encode-failure-octet 8)
|
|
|
432 #x14)))
|
|
|
433 (copy-tree encode-program)))
|
|
|
434 encode-program))
|
|
|
435
|
|
|
436 (defun make-8-bit-create-decode-encode-tables (unicode-map)
|
|
|
437 "Return a list \(DECODE-TABLE ENCODE-TABLE) given UNICODE-MAP.
|
|
|
438 UNICODE-MAP should be an alist mapping from integer octet values to
|
|
|
439 characters with UCS code points; DECODE-TABLE will be a 256-element
|
|
|
440 vector, and ENCODE-TABLE will be a hash table mapping from 256 numbers
|
|
|
441 to 256 distinct characters. "
|
|
|
442 (check-argument-type #'listp unicode-map)
|
|
|
443 (let ((decode-table (make-vector 256 nil))
|
|
|
444 (encode-table (make-hash-table :size 256))
|
|
|
445 (private-use-start (encode-char make-8-bit-private-use-start 'ucs))
|
|
|
446 desired-ucs)
|
|
|
447
|
|
|
448 (loop for (external internal)
|
|
|
449 in unicode-map
|
|
|
450 do
|
|
|
451 (aset decode-table external internal)
|
|
|
452 (assert (not (eq (encode-char internal 'ucs) -1))
|
|
|
453 nil
|
|
|
454 "Looks like you're calling make-8-bit-coding-system in a \
|
|
|
455 dumped file, \nand you're either not providing a literal UNICODE-MAP
|
|
|
456 or PROPS. Don't do that; make-8-bit-coding-system relies on sensible
|
|
|
457 Unicode mappings being available, which they are at compile time for
|
|
|
458 dumped files (but this requires the mentioned literals), but not, for
|
|
|
459 most of them, at run time. ")
|
|
|
460
|
|
|
461 (puthash (encode-char internal 'ucs)
|
|
|
462 ;; This is semantically an integer, but Dave Love's design
|
|
|
463 ;; for lookup-integer in CCL means we need to store it as a
|
|
|
464 ;; character.
|
|
|
465 (int-to-char external)
|
|
|
466 encode-table))
|
|
|
467
|
|
|
468 ;; Now, go through the decode table looking at the characters that
|
|
|
469 ;; remain nil. If the XEmacs character with that integer is already in
|
|
|
470 ;; the encode table, map the on-disk octet to a Unicode private use
|
|
|
471 ;; character. Otherwise map the on-disk octet to the XEmacs character
|
|
|
472 ;; with that numeric value, to make it clearer what it is.
|
|
|
473 (dotimes (i 256)
|
|
|
474 (when (null (aref decode-table i))
|
|
|
475 ;; Find a free code point.
|
|
|
476 (setq desired-ucs i)
|
|
|
477 (while (gethash desired-ucs encode-table)
|
|
|
478 ;; In the normal case, the code point chosen will be U+E0XY, where
|
|
|
479 ;; XY is the hexadecimal octet on disk. In pathological cases
|
|
|
480 ;; it'll be something else.
|
|
|
481 (setq desired-ucs (+ private-use-start desired-ucs)
|
|
|
482 private-use-start (+ private-use-start 1)))
|
|
4085
|
483 (puthash desired-ucs (int-to-char i) encode-table)
|
|
|
484 (setq desired-ucs (if (> desired-ucs #xFF)
|
|
|
485 (decode-char 'ucs desired-ucs)
|
|
|
486 ;; So we get Latin-1 when run at dump time,
|
|
|
487 ;; instead of JIT-allocated characters.
|
|
|
488 (int-to-char desired-ucs)))
|
|
|
489 (aset decode-table i desired-ucs)))
|
|
4072
|
490 (values decode-table encode-table)))
|
|
|
491
|
|
|
492 (defun make-8-bit-generate-decode-program (decode-table)
|
|
|
493 "Given DECODE-TABLE, generate a CCL program to decode an 8-bit charset.
|
|
|
494 DECODE-TABLE must have 256 non-cons entries, and will be regarded as
|
|
|
495 describing a map from the octet corresponding to an offset in the
|
|
|
496 table to the that entry in the table. "
|
|
|
497 (check-argument-type #'vectorp decode-table)
|
|
|
498 (check-argument-range (length decode-table) #x100 #x100)
|
|
|
499 (let ((decode-program-parts
|
|
|
500 (eval-when-compile
|
|
|
501 (let* ((compiled
|
|
|
502 (append
|
|
|
503 (ccl-compile
|
|
|
504 `(3
|
|
|
505 ((read r0)
|
|
|
506 (loop
|
|
|
507 (write-read-repeat r0 ,(make-vector
|
|
|
508 256 'sentinel)))))) nil))
|
|
|
509 (first-part compiled)
|
|
|
510 (last-part
|
|
|
511 (member-if-not #'symbolp
|
|
|
512 (member-if-not #'integerp first-part))))
|
|
|
513 ;; Chop off the sentinel sentinel sentinel [..] part.
|
|
|
514 (while compiled
|
|
|
515 (if (symbolp (cadr compiled))
|
|
|
516 (setcdr compiled nil))
|
|
|
517 (setq compiled (cdr compiled)))
|
|
|
518 (list first-part last-part)))))
|
|
|
519 (nconc
|
|
|
520 ;; copy-list needed, because the structure of the literal provided
|
|
|
521 ;; by our eval-when-compile hangs around.
|
|
|
522 (copy-list (first decode-program-parts))
|
|
|
523 (append decode-table nil)
|
|
|
524 (second decode-program-parts))))
|
|
|
525
|
|
|
526 ;;;###autoload
|
|
|
527 (defun make-8-bit-coding-system (name unicode-map &optional description props)
|
|
|
528 "Make and return a fixed-width 8-bit CCL coding system named NAME.
|
|
|
529 NAME must be a symbol, and UNICODE-MAP a list.
|
|
|
530
|
|
|
531 UNICODE-MAP is a plist describing a map from octets in the coding
|
|
|
532 system NAME (as integers) to XEmacs characters. Those XEmacs
|
|
|
533 characters will be used explicitly on decoding, but for encoding (most
|
|
|
534 relevantly, on writing to disk) XEmacs characters that map to the same
|
|
|
535 Unicode code point will be unified. This means that the ISO-8859-?
|
|
|
536 characters that map to the same Unicode code point will not be
|
|
|
537 distinct when written to disk, which is normally what is intended; it
|
|
|
538 also means that East Asian Han characters from different XEmacs
|
|
|
539 character sets will not be distinct when written to disk, which is
|
|
|
540 less often what is intended.
|
|
|
541
|
|
|
542 Any octets not mapped will be decoded into the ISO 8859-1 characters with
|
|
|
543 the corresponding numeric value; unless another octet maps to that
|
|
|
544 character, in which case the Unicode private use area will be used. This
|
|
|
545 avoids spurious changes to files on disk when they contain octets that would
|
|
|
546 be otherwise remapped to the canonical values for the corresponding
|
|
|
547 characters in the coding system.
|
|
|
548
|
|
|
549 DESCRIPTION and PROPS are as in `make-coding-system', which see. This
|
|
|
550 function also accepts two additional (optional) properties in PROPS;
|
|
|
551 `aliases', giving a list of aliases to be initialized for this
|
|
|
552 coding-system, and `encode-failure-octet', an integer between 0 and 256 to
|
|
|
553 write in place of XEmacs characters that cannot be encoded, defaulting to
|
|
|
554 the code for tilde `~'. "
|
|
|
555 (check-argument-type #'symbolp name)
|
|
|
556 (check-argument-type #'listp unicode-map)
|
|
|
557 (check-argument-type #'stringp
|
|
|
558 (or description
|
|
|
559 (setq description
|
|
|
560 (format "Coding system used for %s." name))))
|
|
|
561 (check-valid-plist props)
|
|
|
562 (let ((encode-failure-octet (or (plist-get props 'encode-failure-octet)
|
|
|
563 (char-to-int ?~)))
|
|
|
564 (aliases (plist-get props 'aliases))
|
|
|
565 (hash-table-sym (gentemp (format "%s-encode-table" name)))
|
|
|
566 encode-program decode-program result decode-table encode-table)
|
|
|
567
|
|
|
568 ;; Some more sanity checking.
|
|
|
569 (check-argument-range encode-failure-octet 0 #xFF)
|
|
|
570 (check-argument-type #'listp aliases)
|
|
|
571
|
|
|
572 ;; Don't pass on our extra data to make-coding-system.
|
|
|
573 (setq props (plist-remprop props 'encode-failure-octet)
|
|
|
574 props (plist-remprop props 'aliases))
|
|
|
575
|
|
|
576 (multiple-value-setq
|
|
|
577 (decode-table encode-table)
|
|
|
578 (make-8-bit-create-decode-encode-tables unicode-map))
|
|
|
579
|
|
|
580 ;; Register the decode-table.
|
|
|
581 (define-translation-hash-table hash-table-sym encode-table)
|
|
|
582
|
|
|
583 ;; Generate the programs.
|
|
|
584 (setq decode-program (make-8-bit-generate-decode-program decode-table)
|
|
|
585 encode-program (make-8-bit-generate-encode-program
|
|
|
586 decode-table encode-table encode-failure-octet))
|
|
|
587 (unless (vectorp encode-program)
|
|
|
588 (setq encode-program
|
|
|
589 (apply #'vector
|
|
|
590 (nsublis (list (cons 'encode-table-sym hash-table-sym))
|
|
|
591 (copy-tree encode-program)))))
|
|
|
592 (unless (vectorp decode-program)
|
|
|
593 (setq decode-program
|
|
|
594 (apply #'vector decode-program)))
|
|
|
595
|
|
|
596 ;; And now generate the actual coding system.
|
|
|
597 (setq result
|
|
|
598 (make-coding-system
|
|
|
599 name 'ccl
|
|
|
600 description
|
|
|
601 (plist-put (plist-put props 'decode decode-program)
|
|
|
602 'encode encode-program)))
|
|
|
603 (coding-system-put name 'category 'iso-8-1)
|
|
|
604 (loop for alias in aliases
|
|
|
605 do (define-coding-system-alias alias name))
|
|
|
606 result))
|
|
|
607
|
|
|
608 ;;;###autoload
|
|
|
609 (define-compiler-macro make-8-bit-coding-system (&whole form name unicode-map
|
|
|
610 &optional description props)
|
|
|
611
|
|
|
612 ;; We provide the compiler macro (= macro that is expanded only on
|
|
|
613 ;; compilation, and that can punt to a runtime version of the
|
|
|
614 ;; associate function if necessary) not for reasons of speed, though
|
|
|
615 ;; it does speed up things at runtime a little, but because the
|
|
|
616 ;; Unicode mappings are available at compile time in the dumped
|
|
|
617 ;; files, but they are not available at run time for the vast
|
|
|
618 ;; majority of them.
|
|
|
619
|
|
|
620 (if (not (and (and (consp name) (eq (car name) 'quote))
|
|
|
621 (and (consp unicode-map) (eq (car unicode-map) 'quote))
|
|
|
622 (and (or (and (consp props) (eq (car props) 'quote))
|
|
|
623 (null props)))))
|
|
|
624 ;; The call does not use literals; do it at runtime.
|
|
|
625 form
|
|
|
626 (setq name (cadr name)
|
|
|
627 unicode-map (cadr unicode-map)
|
|
|
628 props (if props (cadr props)))
|
|
|
629 (let ((encode-failure-octet
|
|
|
630 (or (plist-get props 'encode-failure-octet) (char-to-int ?~)))
|
|
|
631 (aliases (plist-get props 'aliases))
|
|
|
632 encode-program decode-program
|
|
|
633 decode-table encode-table res)
|
|
|
634
|
|
|
635 ;; Some sanity checking.
|
|
|
636 (check-argument-range encode-failure-octet 0 #xFF)
|
|
|
637 (check-argument-type #'listp aliases)
|
|
|
638
|
|
|
639 ;; Don't pass on our extra data to make-coding-system.
|
|
|
640 (setq props (plist-remprop props 'encode-failure-octet)
|
|
|
641 props (plist-remprop props 'aliases))
|
|
|
642
|
|
|
643 ;; Work out encode-table and decode-table.
|
|
|
644 (multiple-value-setq
|
|
|
645 (decode-table encode-table)
|
|
|
646 (make-8-bit-create-decode-encode-tables unicode-map))
|
|
|
647
|
|
|
648 ;; Generate the decode and encode programs.
|
|
|
649 (setq decode-program (make-8-bit-generate-decode-program decode-table)
|
|
|
650 encode-program (make-8-bit-generate-encode-program
|
|
|
651 decode-table encode-table encode-failure-octet))
|
|
|
652
|
|
|
653 ;; And return the generated code.
|
|
|
654 `(let ((encode-table-sym (gentemp (format "%s-encode-table" ',name)))
|
|
|
655 result)
|
|
|
656 (define-translation-hash-table encode-table-sym ,encode-table)
|
|
|
657 (setq result
|
|
|
658 (make-coding-system
|
|
|
659 ',name 'ccl ,description
|
|
|
660 (plist-put (plist-put ',props 'decode
|
|
|
661 ,(apply #'vector decode-program))
|
|
|
662 'encode
|
|
|
663 (apply #'vector
|
|
|
664 (nsublis
|
|
|
665 (list (cons
|
|
|
666 'encode-table-sym
|
|
|
667 (symbol-value 'encode-table-sym)))
|
|
|
668 ',encode-program)))))
|
|
|
669 (coding-system-put ',name 'category 'iso-8-1)
|
|
|
670 ,(macroexpand `(loop for alias in ',aliases
|
|
|
671 do (define-coding-system-alias alias
|
|
|
672 ',name)))
|
|
|
673 'result))))
|
|
|
674
|
|
|
675 |
