|
502
|
1 ;;; mule-coding.el --- Coding-system functions for Mule. -*- coding: iso-2022-7bit; -*-
|
|
333
|
2
|
|
|
3 ;; Copyright (C) 1995 Electrotechnical Laboratory, JAPAN.
|
|
|
4 ;; Licensed to the Free Software Foundation.
|
|
|
5 ;; Copyright (C) 1995 Amdahl Corporation.
|
|
|
6 ;; Copyright (C) 1995 Sun Microsystems.
|
|
|
7 ;; Copyright (C) 1997 MORIOKA Tomohiko
|
|
771
|
8 ;; Copyright (C) 2001 Ben Wing.
|
|
333
|
9
|
|
|
10 ;; This file is part of XEmacs.
|
|
|
11
|
|
|
12 ;; XEmacs is free software; you can redistribute it and/or modify it
|
|
|
13 ;; under the terms of the GNU General Public License as published by
|
|
|
14 ;; the Free Software Foundation; either version 2, or (at your option)
|
|
|
15 ;; any later version.
|
|
|
16
|
|
|
17 ;; XEmacs is distributed in the hope that it will be useful, but
|
|
|
18 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
20 ;; General Public License for more details.
|
|
|
21
|
|
|
22 ;; You should have received a copy of the GNU General Public License
|
|
444
|
23 ;; along with XEmacs; see the file COPYING. If not, write to the
|
|
333
|
24 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
|
25 ;; Boston, MA 02111-1307, USA.
|
|
|
26
|
|
|
27 ;;; Commentary:
|
|
|
28
|
|
|
29 ;;; split off of mule.el and mostly moved to coding.el
|
|
|
30
|
|
4072
|
31 ;; Needed for make-8-bit-coding-system.
|
|
4080
|
32 (eval-when-compile (require 'ccl))
|
|
4072
|
33
|
|
333
|
34 ;;; Code:
|
|
|
35
|
|
|
36 (defun coding-system-force-on-output (coding-system register)
|
|
|
37 "Return the 'force-on-output property of CODING-SYSTEM for the specified REGISTER."
|
|
444
|
38 (check-type register integer)
|
|
333
|
39 (coding-system-property
|
|
|
40 coding-system
|
|
|
41 (case register
|
|
|
42 (0 'force-g0-on-output)
|
|
|
43 (1 'force-g1-on-output)
|
|
|
44 (2 'force-g2-on-output)
|
|
|
45 (3 'force-g3-on-output)
|
|
|
46 (t (signal 'args-out-of-range (list register 0 3))))))
|
|
|
47
|
|
|
48 (defun coding-system-short (coding-system)
|
|
|
49 "Return the 'short property of CODING-SYSTEM."
|
|
|
50 (coding-system-property coding-system 'short))
|
|
|
51
|
|
|
52 (defun coding-system-no-ascii-eol (coding-system)
|
|
|
53 "Return the 'no-ascii-eol property of CODING-SYSTEM."
|
|
|
54 (coding-system-property coding-system 'no-ascii-eol))
|
|
|
55
|
|
|
56 (defun coding-system-no-ascii-cntl (coding-system)
|
|
|
57 "Return the 'no-ascii-cntl property of CODING-SYSTEM."
|
|
|
58 (coding-system-property coding-system 'no-ascii-cntl))
|
|
|
59
|
|
|
60 (defun coding-system-seven (coding-system)
|
|
|
61 "Return the 'seven property of CODING-SYSTEM."
|
|
|
62 (coding-system-property coding-system 'seven))
|
|
|
63
|
|
|
64 (defun coding-system-lock-shift (coding-system)
|
|
|
65 "Return the 'lock-shift property of CODING-SYSTEM."
|
|
|
66 (coding-system-property coding-system 'lock-shift))
|
|
|
67
|
|
|
68 ;;(defun coding-system-use-japanese-jisx0201-roman (coding-system)
|
|
|
69 ;; "Return the 'use-japanese-jisx0201-roman property of CODING-SYSTEM."
|
|
|
70 ;; (coding-system-property coding-system 'use-japanese-jisx0201-roman))
|
|
|
71
|
|
|
72 ;;(defun coding-system-use-japanese-jisx0208-1978 (coding-system)
|
|
|
73 ;; "Return the 'use-japanese-jisx0208-1978 property of CODING-SYSTEM."
|
|
|
74 ;; (coding-system-property coding-system 'use-japanese-jisx0208-2978))
|
|
|
75
|
|
|
76 (defun coding-system-no-iso6429 (coding-system)
|
|
|
77 "Return the 'no-iso6429 property of CODING-SYSTEM."
|
|
|
78 (coding-system-property coding-system 'no-iso6429))
|
|
|
79
|
|
|
80 (defun coding-system-ccl-encode (coding-system)
|
|
|
81 "Return the CCL 'encode property of CODING-SYSTEM."
|
|
|
82 (coding-system-property coding-system 'encode))
|
|
|
83
|
|
|
84 (defun coding-system-ccl-decode (coding-system)
|
|
|
85 "Return the CCL 'decode property of CODING-SYSTEM."
|
|
|
86 (coding-system-property coding-system 'decode))
|
|
|
87
|
|
771
|
88 (defun coding-system-iso2022-charset (coding-system register)
|
|
|
89 "Return the charset initially designated to REGISTER in CODING-SYSTEM.
|
|
|
90 The allowable range of REGISTER is 0 through 3."
|
|
|
91 (if (or (< register 0) (> register 3))
|
|
|
92 (error 'args-out-of-range "coding-system-charset REGISTER" register 0 3))
|
|
|
93 (coding-system-property coding-system (nth register '(charset-g0
|
|
|
94 charset-g1
|
|
|
95 charset-g2
|
|
|
96 charset-g3))))
|
|
|
97
|
|
333
|
98
|
|
|
99 ;;;; Definitions of predefined coding systems
|
|
|
100
|
|
|
101 (make-coding-system
|
|
|
102 'ctext 'iso2022
|
|
771
|
103 "Compound Text"
|
|
333
|
104 '(charset-g0 ascii
|
|
|
105 charset-g1 latin-iso8859-1
|
|
|
106 eol-type nil
|
|
|
107 mnemonic "CText"))
|
|
|
108
|
|
|
109 (make-coding-system
|
|
|
110 'iso-8859-1 'no-conversion
|
|
771
|
111 "ISO-8859-1 (Latin-1)"
|
|
333
|
112 '(eol-type nil mnemonic "Noconv"))
|
|
|
113
|
|
|
114 (make-coding-system
|
|
|
115 'iso-2022-8bit-ss2 'iso2022
|
|
771
|
116 "ISO-2022 8-bit w/SS2"
|
|
333
|
117 '(charset-g0 ascii
|
|
|
118 charset-g1 latin-iso8859-1
|
|
|
119 charset-g2 t ;; unspecified but can be used later.
|
|
|
120 short t
|
|
|
121 mnemonic "ISO8/SS"
|
|
771
|
122 documentation "ISO 2022 based 8-bit encoding using SS2 for 96-charset"
|
|
333
|
123 ))
|
|
|
124
|
|
|
125 (make-coding-system
|
|
|
126 'iso-2022-7bit-ss2 'iso2022
|
|
771
|
127 "ISO-2022 7-bit w/SS2"
|
|
333
|
128 '(charset-g0 ascii
|
|
|
129 charset-g2 t ;; unspecified but can be used later.
|
|
|
130 seven t
|
|
|
131 short t
|
|
|
132 mnemonic "ISO7/SS"
|
|
771
|
133 documentation "ISO 2022 based 7-bit encoding using SS2 for 96-charset"
|
|
333
|
134 eol-type nil))
|
|
|
135
|
|
|
136 ;; (copy-coding-system 'iso-2022-7bit-ss2 'iso-2022-jp-2)
|
|
|
137 (make-coding-system
|
|
|
138 'iso-2022-jp-2 'iso2022
|
|
771
|
139 "ISO-2022-JP-2"
|
|
333
|
140 '(charset-g0 ascii
|
|
|
141 charset-g2 t ;; unspecified but can be used later.
|
|
|
142 seven t
|
|
|
143 short t
|
|
|
144 mnemonic "ISO7/SS"
|
|
|
145 eol-type nil))
|
|
|
146
|
|
|
147 (make-coding-system
|
|
|
148 'iso-2022-7bit 'iso2022
|
|
771
|
149 "ISO 2022 7-bit"
|
|
333
|
150 '(charset-g0 ascii
|
|
|
151 seven t
|
|
|
152 short t
|
|
771
|
153 mnemonic "ISO7"
|
|
|
154 documentation "ISO-2022-based 7-bit encoding using only G0"
|
|
|
155 ))
|
|
333
|
156
|
|
|
157 ;; compatibility for old XEmacsen
|
|
771
|
158 (define-coding-system-alias 'iso-2022-7 'iso-2022-7bit)
|
|
333
|
159
|
|
|
160 (make-coding-system
|
|
|
161 'iso-2022-8 'iso2022
|
|
771
|
162 "ISO-2022 8-bit"
|
|
333
|
163 '(charset-g0 ascii
|
|
|
164 charset-g1 latin-iso8859-1
|
|
|
165 short t
|
|
|
166 mnemonic "ISO8"
|
|
771
|
167 documentation "ISO-2022 eight-bit coding system. No single-shift or locking-shift."
|
|
333
|
168 ))
|
|
|
169
|
|
|
170 (make-coding-system
|
|
|
171 'escape-quoted 'iso2022
|
|
771
|
172 "Escape-Quoted (for .ELC files)"
|
|
333
|
173 '(charset-g0 ascii
|
|
|
174 charset-g1 latin-iso8859-1
|
|
|
175 eol-type lf
|
|
|
176 escape-quoted t
|
|
|
177 mnemonic "ESC/Quot"
|
|
771
|
178 documentation "ISO-2022 eight-bit coding system with escape quoting; used for .ELC files."
|
|
333
|
179 ))
|
|
|
180
|
|
|
181 (make-coding-system
|
|
|
182 'iso-2022-lock 'iso2022
|
|
771
|
183 "ISO-2022 w/locking-shift"
|
|
333
|
184 '(charset-g0 ascii
|
|
|
185 charset-g1 t ;; unspecified but can be used later.
|
|
|
186 seven t
|
|
|
187 lock-shift t
|
|
|
188 mnemonic "ISO7/Lock"
|
|
771
|
189 documentation "ISO-2022 coding system using Locking-Shift for 96-charset."
|
|
333
|
190 ))
|
|
4072
|
191
|
|
333
|
192
|
|
4072
|
193 ;; This is used by people writing CCL programs, but is called at runtime.
|
|
|
194 (defun define-translation-hash-table (symbol table)
|
|
|
195 "Define SYMBOL as the name of the hash translation TABLE for use in CCL.
|
|
|
196
|
|
|
197 Analogous to `define-translation-table', but updates
|
|
|
198 `translation-hash-table-vector' and the table is for use in the CCL
|
|
|
199 `lookup-integer' and `lookup-character' functions."
|
|
4145
|
200 (check-argument-type #'symbolp symbol)
|
|
|
201 (check-argument-type #'hash-table-p table)
|
|
4072
|
202 (let ((len (length translation-hash-table-vector))
|
|
|
203 (id 0)
|
|
|
204 done)
|
|
|
205 (put symbol 'translation-hash-table table)
|
|
|
206 (while (not done)
|
|
|
207 (if (>= id len)
|
|
|
208 (setq translation-hash-table-vector
|
|
|
209 (vconcat translation-hash-table-vector [nil])))
|
|
|
210 (let ((slot (aref translation-hash-table-vector id)))
|
|
|
211 (if (or (not slot)
|
|
|
212 (eq (car slot) symbol))
|
|
|
213 (progn
|
|
|
214 (aset translation-hash-table-vector id (cons symbol table))
|
|
|
215 (setq done t))
|
|
|
216 (setq id (1+ id)))))
|
|
|
217 (put symbol 'translation-hash-table-id id)
|
|
|
218 id))
|
|
|
219
|
|
|
220 (defvar make-8-bit-private-use-start (decode-char 'ucs #xE000)
|
|
|
221 "Start of a 256 code private use area for make-8-bit-coding-system.
|
|
|
222
|
|
|
223 This is used to ensure that distinct octets on disk for a given coding
|
|
|
224 system map to distinct XEmacs characters, preventing a spurious changes when
|
|
|
225 a file is read, not changed, and then written. ")
|
|
|
226
|
|
|
227 (defun make-8-bit-generate-helper (decode-table encode-table
|
|
|
228 encode-failure-octet)
|
|
|
229 "Helper function for `make-8-bit-generate-encode-program', which see.
|
|
|
230
|
|
4145
|
231 Deals with the case where ASCII and another character set can both be
|
|
|
232 encoded unambiguously and completely into the coding-system; if this is so,
|
|
|
233 returns a list corresponding to such a ccl-program. If not, it returns nil. "
|
|
4072
|
234 (let ((tentative-encode-program-parts
|
|
|
235 (eval-when-compile
|
|
4295
|
236 (let* ((vec-len 128)
|
|
|
237 (compiled
|
|
4072
|
238 (append
|
|
|
239 (ccl-compile
|
|
|
240 `(1
|
|
|
241 (loop
|
|
|
242 (read-multibyte-character r0 r1)
|
|
|
243 (if (r0 == ,(charset-id 'ascii))
|
|
|
244 (write r1)
|
|
|
245 ((if (r0 == #xABAB)
|
|
|
246 ;; #xBFFE is a sentinel in the compiled
|
|
|
247 ;; program.
|
|
4295
|
248 ;; #xBFFE is a sentinel in the compiled
|
|
|
249 ;; program.
|
|
|
250 ((r0 = r1 & #x7F)
|
|
|
251 (write r0 ,(make-vector vec-len #xBFFE)))
|
|
4072
|
252 ((mule-to-unicode r0 r1)
|
|
|
253 (if (r0 == #xFFFD)
|
|
|
254 (write #xBEEF)
|
|
|
255 ((lookup-integer encode-table-sym r0 r3)
|
|
|
256 (if r7
|
|
|
257 (write-multibyte-character r0 r3)
|
|
|
258 (write #xBEEF))))))))
|
|
|
259 (repeat)))) nil))
|
|
|
260 (first-part compiled)
|
|
|
261 (last-part
|
|
|
262 (member-if-not (lambda (entr) (eq #xBFFE entr))
|
|
|
263 (member-if
|
|
|
264 (lambda (entr) (eq #xBFFE entr))
|
|
|
265 first-part))))
|
|
|
266 (while compiled
|
|
4295
|
267 (when (eq #xBFFE (cadr compiled))
|
|
|
268 (assert (= vec-len (search '(#xBFFE) (cdr compiled)
|
|
|
269 :test #'/=)) nil
|
|
|
270 "Strange ccl vector length")
|
|
|
271 (setcdr compiled nil))
|
|
4072
|
272 (setq compiled (cdr compiled)))
|
|
|
273 ;; Is the generated code as we expect it to be?
|
|
|
274 (assert (and (memq #xABAB first-part)
|
|
|
275 (memq #xBEEF14 last-part))
|
|
|
276 nil
|
|
|
277 "This code assumes that the constant #xBEEF is #xBEEF14 in \
|
|
|
278 compiled CCL code,\nand that the constant #xABAB is #xABAB. If that is
|
|
|
279 not the case, and it appears not to be--that's why you're getting this
|
|
|
280 message--it will not work. ")
|
|
4295
|
281 (list first-part last-part vec-len))))
|
|
4072
|
282 (charset-lower -1)
|
|
|
283 (charset-upper -1)
|
|
|
284 worth-trying known-charsets encode-program
|
|
4295
|
285 other-charset-vector ucs args-out-of-range)
|
|
4072
|
286
|
|
|
287 (loop for char across decode-table
|
|
|
288 do (pushnew (char-charset char) known-charsets))
|
|
|
289 (setq known-charsets (delq 'ascii known-charsets))
|
|
|
290
|
|
|
291 (loop for known-charset in known-charsets
|
|
|
292 do
|
|
|
293 ;; This is not possible for two dimensional charsets.
|
|
|
294 (when (eq 1 (charset-dimension known-charset))
|
|
|
295 (setq args-out-of-range t)
|
|
|
296 (if (eq 'control-1 known-charset)
|
|
|
297 (setq charset-lower 0
|
|
|
298 charset-upper 31)
|
|
|
299 ;; There should be a nicer way to get the limits here.
|
|
|
300 (condition-case args-out-of-range
|
|
|
301 (make-char known-charset #x100)
|
|
|
302 (args-out-of-range
|
|
|
303 (setq charset-lower (third args-out-of-range)
|
|
|
304 charset-upper (fourth args-out-of-range)))))
|
|
|
305 (loop
|
|
|
306 for i from charset-lower to charset-upper
|
|
|
307 always (and (setq ucs
|
|
|
308 (encode-char (make-char known-charset i) 'ucs))
|
|
|
309 (gethash ucs encode-table))
|
|
|
310 finally (setq worth-trying known-charset))
|
|
|
311
|
|
|
312 ;; Only trying this for one charset at a time, the first find.
|
|
|
313 (when worth-trying (return))
|
|
|
314
|
|
|
315 ;; Okay, this charset is not worth trying, Try the next.
|
|
|
316 (setq charset-lower -1
|
|
|
317 charset-upper -1
|
|
|
318 worth-trying nil)))
|
|
|
319
|
|
|
320 (when worth-trying
|
|
4295
|
321 (setq other-charset-vector
|
|
|
322 (make-vector (third tentative-encode-program-parts)
|
|
|
323 encode-failure-octet))
|
|
4072
|
324 (loop for i from charset-lower to charset-upper
|
|
4090
|
325 do (aset other-charset-vector i
|
|
4072
|
326 (gethash (encode-char (make-char worth-trying i)
|
|
|
327 'ucs) encode-table)))
|
|
|
328 (setq encode-program
|
|
|
329 (nsublis
|
|
|
330 (list (cons #xABAB (charset-id worth-trying)))
|
|
|
331 (nconc
|
|
|
332 (copy-list (first
|
|
|
333 tentative-encode-program-parts))
|
|
|
334 (append other-charset-vector nil)
|
|
|
335 (copy-tree (second
|
|
|
336 tentative-encode-program-parts))))))
|
|
|
337 encode-program))
|
|
|
338
|
|
|
339 (defun make-8-bit-generate-encode-program (decode-table encode-table
|
|
|
340 encode-failure-octet)
|
|
|
341 "Generate a CCL program to decode a 8-bit fixed-width charset.
|
|
|
342
|
|
|
343 DECODE-TABLE must have 256 non-cons entries, and will be regarded as
|
|
|
344 describing a map from the octet corresponding to an offset in the
|
|
|
345 table to the that entry in the table. ENCODE-TABLE is a hash table
|
|
|
346 map from unicode values to characters in the range [0,255].
|
|
|
347 ENCODE-FAILURE-OCTET describes an integer between 0 and 255
|
|
|
348 \(inclusive) to write in the event that a character cannot be encoded. "
|
|
|
349 (check-argument-type #'vectorp decode-table)
|
|
|
350 (check-argument-range (length decode-table) #x100 #x100)
|
|
|
351 (check-argument-type #'hash-table-p encode-table)
|
|
|
352 (check-argument-type #'integerp encode-failure-octet)
|
|
|
353 (check-argument-range encode-failure-octet #x00 #xFF)
|
|
|
354 (let ((encode-program nil)
|
|
|
355 (general-encode-program
|
|
|
356 (eval-when-compile
|
|
|
357 (let ((prog (append
|
|
|
358 (ccl-compile
|
|
|
359 `(1
|
|
|
360 (loop
|
|
|
361 (read-multibyte-character r0 r1)
|
|
|
362 (mule-to-unicode r0 r1)
|
|
|
363 (if (r0 == #xFFFD)
|
|
|
364 (write #xBEEF)
|
|
|
365 ((lookup-integer encode-table-sym r0 r3)
|
|
|
366 (if r7
|
|
|
367 (write-multibyte-character r0 r3)
|
|
|
368 (write #xBEEF))))
|
|
|
369 (repeat)))) nil)))
|
|
|
370 (assert (memq #xBEEF14 prog)
|
|
|
371 nil
|
|
|
372 "This code assumes that the constant #xBEEF is #xBEEF14 \
|
|
|
373 in compiled CCL code.\nIf that is not the case, and it appears not to
|
|
|
374 be--that's why you're getting this message--it will not work. ")
|
|
|
375 prog)))
|
|
|
376 (encode-program-with-ascii-optimisation
|
|
|
377 (eval-when-compile
|
|
|
378 (let ((prog (append
|
|
|
379 (ccl-compile
|
|
|
380 `(1
|
|
|
381 (loop
|
|
|
382 (read-multibyte-character r0 r1)
|
|
|
383 (if (r0 == ,(charset-id 'ascii))
|
|
|
384 (write r1)
|
|
|
385 ((mule-to-unicode r0 r1)
|
|
|
386 (if (r0 == #xFFFD)
|
|
|
387 (write #xBEEF)
|
|
|
388 ((lookup-integer encode-table-sym r0 r3)
|
|
|
389 (if r7
|
|
|
390 (write-multibyte-character r0 r3)
|
|
|
391 (write #xBEEF))))))
|
|
|
392 (repeat)))) nil)))
|
|
|
393 (assert (memq #xBEEF14 prog)
|
|
|
394 nil
|
|
|
395 "This code assumes that the constant #xBEEF is #xBEEF14 \
|
|
|
396 in compiled CCL code.\nIf that is not the case, and it appears not to
|
|
|
397 be--that's why you're getting this message--it will not work. ")
|
|
|
398 prog)))
|
|
|
399 (ascii-encodes-as-itself nil))
|
|
|
400
|
|
|
401 ;; Is this coding system ASCII-compatible? If so, we can avoid the hash
|
|
|
402 ;; table lookup for those characters.
|
|
|
403 (loop
|
|
|
404 for i from #x00 to #x7f
|
|
|
405 always (eq (int-to-char i) (gethash i encode-table))
|
|
|
406 finally (setq ascii-encodes-as-itself t))
|
|
|
407
|
|
|
408 ;; Note that this logic handles EBCDIC badly. For example, CP037,
|
|
|
409 ;; MIME name ebcdic-na, has the entire repertoire of ASCII and
|
|
|
410 ;; Latin 1, and thus a more optimal ccl encode program would check
|
|
|
411 ;; for those character sets and use tables. But for now, we do a
|
|
|
412 ;; hash table lookup for every character.
|
|
|
413 (if (null ascii-encodes-as-itself)
|
|
|
414 ;; General encode program. Pros; general and correct. Cons;
|
|
|
415 ;; slow, a hash table lookup + mule-unicode conversion is done
|
|
|
416 ;; for every character encoding.
|
|
|
417 (setq encode-program general-encode-program)
|
|
|
418 (setq encode-program
|
|
|
419 ;; Encode program with ascii-ascii mapping (based on a
|
|
|
420 ;; character's mule character set), and one other mule
|
|
|
421 ;; character set using table-based encoding, other
|
|
|
422 ;; character sets using hash table lookups.
|
|
|
423 ;; make-8-bit-non-ascii-completely-coveredp only returns
|
|
|
424 ;; such a mapping if some non-ASCII charset with
|
|
|
425 ;; characters in decode-table is entirely covered by
|
|
|
426 ;; encode-table.
|
|
|
427 (make-8-bit-generate-helper decode-table encode-table
|
|
|
428 encode-failure-octet))
|
|
|
429 (unless encode-program
|
|
|
430 ;; If make-8-bit-non-ascii-completely-coveredp returned nil,
|
|
|
431 ;; but ASCII still encodes as itself, do one-to-one mapping
|
|
|
432 ;; for ASCII, and a hash table lookup for everything else.
|
|
|
433 (setq encode-program encode-program-with-ascii-optimisation)))
|
|
|
434
|
|
|
435 (setq encode-program
|
|
|
436 (nsublis
|
|
|
437 (list (cons #xBEEF14
|
|
|
438 (logior (lsh encode-failure-octet 8)
|
|
|
439 #x14)))
|
|
|
440 (copy-tree encode-program)))
|
|
|
441 encode-program))
|
|
|
442
|
|
|
443 (defun make-8-bit-create-decode-encode-tables (unicode-map)
|
|
|
444 "Return a list \(DECODE-TABLE ENCODE-TABLE) given UNICODE-MAP.
|
|
|
445 UNICODE-MAP should be an alist mapping from integer octet values to
|
|
|
446 characters with UCS code points; DECODE-TABLE will be a 256-element
|
|
|
447 vector, and ENCODE-TABLE will be a hash table mapping from 256 numbers
|
|
|
448 to 256 distinct characters. "
|
|
|
449 (check-argument-type #'listp unicode-map)
|
|
|
450 (let ((decode-table (make-vector 256 nil))
|
|
|
451 (encode-table (make-hash-table :size 256))
|
|
|
452 (private-use-start (encode-char make-8-bit-private-use-start 'ucs))
|
|
|
453 desired-ucs)
|
|
|
454
|
|
|
455 (loop for (external internal)
|
|
|
456 in unicode-map
|
|
|
457 do
|
|
|
458 (aset decode-table external internal)
|
|
|
459 (assert (not (eq (encode-char internal 'ucs) -1))
|
|
|
460 nil
|
|
|
461 "Looks like you're calling make-8-bit-coding-system in a \
|
|
|
462 dumped file, \nand you're either not providing a literal UNICODE-MAP
|
|
|
463 or PROPS. Don't do that; make-8-bit-coding-system relies on sensible
|
|
|
464 Unicode mappings being available, which they are at compile time for
|
|
|
465 dumped files (but this requires the mentioned literals), but not, for
|
|
|
466 most of them, at run time. ")
|
|
|
467
|
|
|
468 (puthash (encode-char internal 'ucs)
|
|
|
469 ;; This is semantically an integer, but Dave Love's design
|
|
|
470 ;; for lookup-integer in CCL means we need to store it as a
|
|
|
471 ;; character.
|
|
|
472 (int-to-char external)
|
|
|
473 encode-table))
|
|
|
474
|
|
|
475 ;; Now, go through the decode table looking at the characters that
|
|
|
476 ;; remain nil. If the XEmacs character with that integer is already in
|
|
|
477 ;; the encode table, map the on-disk octet to a Unicode private use
|
|
|
478 ;; character. Otherwise map the on-disk octet to the XEmacs character
|
|
|
479 ;; with that numeric value, to make it clearer what it is.
|
|
|
480 (dotimes (i 256)
|
|
|
481 (when (null (aref decode-table i))
|
|
|
482 ;; Find a free code point.
|
|
|
483 (setq desired-ucs i)
|
|
|
484 (while (gethash desired-ucs encode-table)
|
|
|
485 ;; In the normal case, the code point chosen will be U+E0XY, where
|
|
|
486 ;; XY is the hexadecimal octet on disk. In pathological cases
|
|
|
487 ;; it'll be something else.
|
|
|
488 (setq desired-ucs (+ private-use-start desired-ucs)
|
|
|
489 private-use-start (+ private-use-start 1)))
|
|
4085
|
490 (puthash desired-ucs (int-to-char i) encode-table)
|
|
|
491 (setq desired-ucs (if (> desired-ucs #xFF)
|
|
|
492 (decode-char 'ucs desired-ucs)
|
|
|
493 ;; So we get Latin-1 when run at dump time,
|
|
|
494 ;; instead of JIT-allocated characters.
|
|
|
495 (int-to-char desired-ucs)))
|
|
|
496 (aset decode-table i desired-ucs)))
|
|
4072
|
497 (values decode-table encode-table)))
|
|
|
498
|
|
|
499 (defun make-8-bit-generate-decode-program (decode-table)
|
|
|
500 "Given DECODE-TABLE, generate a CCL program to decode an 8-bit charset.
|
|
|
501 DECODE-TABLE must have 256 non-cons entries, and will be regarded as
|
|
|
502 describing a map from the octet corresponding to an offset in the
|
|
|
503 table to the that entry in the table. "
|
|
|
504 (check-argument-type #'vectorp decode-table)
|
|
|
505 (check-argument-range (length decode-table) #x100 #x100)
|
|
|
506 (let ((decode-program-parts
|
|
|
507 (eval-when-compile
|
|
|
508 (let* ((compiled
|
|
|
509 (append
|
|
|
510 (ccl-compile
|
|
|
511 `(3
|
|
|
512 ((read r0)
|
|
|
513 (loop
|
|
|
514 (write-read-repeat r0 ,(make-vector
|
|
|
515 256 'sentinel)))))) nil))
|
|
|
516 (first-part compiled)
|
|
|
517 (last-part
|
|
|
518 (member-if-not #'symbolp
|
|
|
519 (member-if-not #'integerp first-part))))
|
|
|
520 ;; Chop off the sentinel sentinel sentinel [..] part.
|
|
|
521 (while compiled
|
|
|
522 (if (symbolp (cadr compiled))
|
|
|
523 (setcdr compiled nil))
|
|
|
524 (setq compiled (cdr compiled)))
|
|
|
525 (list first-part last-part)))))
|
|
|
526 (nconc
|
|
|
527 ;; copy-list needed, because the structure of the literal provided
|
|
|
528 ;; by our eval-when-compile hangs around.
|
|
|
529 (copy-list (first decode-program-parts))
|
|
|
530 (append decode-table nil)
|
|
|
531 (second decode-program-parts))))
|
|
|
532
|
|
4145
|
533 (defun make-8-bit-choose-category (decode-table)
|
|
|
534 "Given DECODE-TABLE, return an appropriate coding category.
|
|
|
535 DECODE-TABLE is a 256-entry vector describing the mapping from octets on
|
|
|
536 disk to XEmacs characters for some fixed-width 8-bit coding system. "
|
|
|
537 (check-argument-type #'vectorp decode-table)
|
|
|
538 (check-argument-range (length decode-table) #x100 #x100)
|
|
|
539 (block category
|
|
|
540 (loop
|
|
|
541 for i from #x80 to #xBF
|
|
|
542 do (unless (= i (aref decode-table i))
|
|
|
543 (return-from category 'no-conversion)))
|
|
|
544 'iso-8-1))
|
|
|
545
|
|
4072
|
546 ;;;###autoload
|
|
|
547 (defun make-8-bit-coding-system (name unicode-map &optional description props)
|
|
|
548 "Make and return a fixed-width 8-bit CCL coding system named NAME.
|
|
|
549 NAME must be a symbol, and UNICODE-MAP a list.
|
|
|
550
|
|
|
551 UNICODE-MAP is a plist describing a map from octets in the coding
|
|
|
552 system NAME (as integers) to XEmacs characters. Those XEmacs
|
|
|
553 characters will be used explicitly on decoding, but for encoding (most
|
|
|
554 relevantly, on writing to disk) XEmacs characters that map to the same
|
|
|
555 Unicode code point will be unified. This means that the ISO-8859-?
|
|
|
556 characters that map to the same Unicode code point will not be
|
|
|
557 distinct when written to disk, which is normally what is intended; it
|
|
|
558 also means that East Asian Han characters from different XEmacs
|
|
|
559 character sets will not be distinct when written to disk, which is
|
|
|
560 less often what is intended.
|
|
|
561
|
|
|
562 Any octets not mapped will be decoded into the ISO 8859-1 characters with
|
|
|
563 the corresponding numeric value; unless another octet maps to that
|
|
|
564 character, in which case the Unicode private use area will be used. This
|
|
|
565 avoids spurious changes to files on disk when they contain octets that would
|
|
|
566 be otherwise remapped to the canonical values for the corresponding
|
|
|
567 characters in the coding system.
|
|
|
568
|
|
|
569 DESCRIPTION and PROPS are as in `make-coding-system', which see. This
|
|
|
570 function also accepts two additional (optional) properties in PROPS;
|
|
|
571 `aliases', giving a list of aliases to be initialized for this
|
|
|
572 coding-system, and `encode-failure-octet', an integer between 0 and 256 to
|
|
|
573 write in place of XEmacs characters that cannot be encoded, defaulting to
|
|
|
574 the code for tilde `~'. "
|
|
|
575 (check-argument-type #'symbolp name)
|
|
|
576 (check-argument-type #'listp unicode-map)
|
|
|
577 (check-argument-type #'stringp
|
|
|
578 (or description
|
|
|
579 (setq description
|
|
|
580 (format "Coding system used for %s." name))))
|
|
|
581 (check-valid-plist props)
|
|
|
582 (let ((encode-failure-octet (or (plist-get props 'encode-failure-octet)
|
|
|
583 (char-to-int ?~)))
|
|
|
584 (aliases (plist-get props 'aliases))
|
|
|
585 (hash-table-sym (gentemp (format "%s-encode-table" name)))
|
|
|
586 encode-program decode-program result decode-table encode-table)
|
|
|
587
|
|
|
588 ;; Some more sanity checking.
|
|
|
589 (check-argument-range encode-failure-octet 0 #xFF)
|
|
|
590 (check-argument-type #'listp aliases)
|
|
|
591
|
|
|
592 ;; Don't pass on our extra data to make-coding-system.
|
|
|
593 (setq props (plist-remprop props 'encode-failure-octet)
|
|
|
594 props (plist-remprop props 'aliases))
|
|
|
595
|
|
|
596 (multiple-value-setq
|
|
|
597 (decode-table encode-table)
|
|
|
598 (make-8-bit-create-decode-encode-tables unicode-map))
|
|
|
599
|
|
|
600 ;; Register the decode-table.
|
|
|
601 (define-translation-hash-table hash-table-sym encode-table)
|
|
|
602
|
|
|
603 ;; Generate the programs.
|
|
|
604 (setq decode-program (make-8-bit-generate-decode-program decode-table)
|
|
|
605 encode-program (make-8-bit-generate-encode-program
|
|
|
606 decode-table encode-table encode-failure-octet))
|
|
|
607 (unless (vectorp encode-program)
|
|
|
608 (setq encode-program
|
|
|
609 (apply #'vector
|
|
|
610 (nsublis (list (cons 'encode-table-sym hash-table-sym))
|
|
|
611 (copy-tree encode-program)))))
|
|
|
612 (unless (vectorp decode-program)
|
|
|
613 (setq decode-program
|
|
|
614 (apply #'vector decode-program)))
|
|
|
615
|
|
|
616 ;; And now generate the actual coding system.
|
|
|
617 (setq result
|
|
|
618 (make-coding-system
|
|
|
619 name 'ccl
|
|
|
620 description
|
|
|
621 (plist-put (plist-put props 'decode decode-program)
|
|
|
622 'encode encode-program)))
|
|
4295
|
623 (coding-system-put name '8-bit-fixed t)
|
|
4145
|
624 (coding-system-put name 'category
|
|
|
625 (make-8-bit-choose-category decode-table))
|
|
4072
|
626 (loop for alias in aliases
|
|
|
627 do (define-coding-system-alias alias name))
|
|
|
628 result))
|
|
|
629
|
|
|
630 (define-compiler-macro make-8-bit-coding-system (&whole form name unicode-map
|
|
|
631 &optional description props)
|
|
|
632
|
|
|
633 ;; We provide the compiler macro (= macro that is expanded only on
|
|
|
634 ;; compilation, and that can punt to a runtime version of the
|
|
|
635 ;; associate function if necessary) not for reasons of speed, though
|
|
|
636 ;; it does speed up things at runtime a little, but because the
|
|
|
637 ;; Unicode mappings are available at compile time in the dumped
|
|
|
638 ;; files, but they are not available at run time for the vast
|
|
|
639 ;; majority of them.
|
|
|
640
|
|
|
641 (if (not (and (and (consp name) (eq (car name) 'quote))
|
|
|
642 (and (consp unicode-map) (eq (car unicode-map) 'quote))
|
|
|
643 (and (or (and (consp props) (eq (car props) 'quote))
|
|
|
644 (null props)))))
|
|
|
645 ;; The call does not use literals; do it at runtime.
|
|
|
646 form
|
|
|
647 (setq name (cadr name)
|
|
|
648 unicode-map (cadr unicode-map)
|
|
|
649 props (if props (cadr props)))
|
|
|
650 (let ((encode-failure-octet
|
|
|
651 (or (plist-get props 'encode-failure-octet) (char-to-int ?~)))
|
|
|
652 (aliases (plist-get props 'aliases))
|
|
|
653 encode-program decode-program
|
|
4103
|
654 decode-table encode-table)
|
|
4072
|
655
|
|
|
656 ;; Some sanity checking.
|
|
|
657 (check-argument-range encode-failure-octet 0 #xFF)
|
|
|
658 (check-argument-type #'listp aliases)
|
|
|
659
|
|
|
660 ;; Don't pass on our extra data to make-coding-system.
|
|
|
661 (setq props (plist-remprop props 'encode-failure-octet)
|
|
|
662 props (plist-remprop props 'aliases))
|
|
|
663
|
|
|
664 ;; Work out encode-table and decode-table.
|
|
|
665 (multiple-value-setq
|
|
|
666 (decode-table encode-table)
|
|
|
667 (make-8-bit-create-decode-encode-tables unicode-map))
|
|
|
668
|
|
|
669 ;; Generate the decode and encode programs.
|
|
|
670 (setq decode-program (make-8-bit-generate-decode-program decode-table)
|
|
|
671 encode-program (make-8-bit-generate-encode-program
|
|
|
672 decode-table encode-table encode-failure-octet))
|
|
|
673
|
|
|
674 ;; And return the generated code.
|
|
|
675 `(let ((encode-table-sym (gentemp (format "%s-encode-table" ',name)))
|
|
4103
|
676 ;; The case-fold-search bind shouldn't be necessary. If I take
|
|
|
677 ;; it, out, though, I get:
|
|
|
678 ;;
|
|
|
679 ;; (invalid-read-syntax "Multiply defined symbol label" 1)
|
|
|
680 ;;
|
|
|
681 ;; when the file is byte compiled.
|
|
|
682 (case-fold-search t))
|
|
4072
|
683 (define-translation-hash-table encode-table-sym ,encode-table)
|
|
4103
|
684 (make-coding-system
|
|
|
685 ',name 'ccl ,description
|
|
|
686 (plist-put (plist-put ',props 'decode
|
|
|
687 ,(apply #'vector decode-program))
|
|
|
688 'encode
|
|
|
689 (apply #'vector
|
|
|
690 (nsublis
|
|
|
691 (list (cons
|
|
|
692 'encode-table-sym
|
|
|
693 (symbol-value 'encode-table-sym)))
|
|
|
694 ',encode-program))))
|
|
4295
|
695 (coding-system-put ',name '8-bit-fixed t)
|
|
4145
|
696 (coding-system-put ',name 'category ',
|
|
|
697 (make-8-bit-choose-category decode-table))
|
|
4072
|
698 ,(macroexpand `(loop for alias in ',aliases
|
|
|
699 do (define-coding-system-alias alias
|
|
|
700 ',name)))
|
|
4103
|
701 (find-coding-system ',name)))))
|
