comparison src/file-coding.h @ 259:11cf20601dec r20-5b28

Import from CVS: tag r20-5b28
author cvs
date Mon, 13 Aug 2007 10:23:02 +0200
parents
children 405dd6d1825b
comparison
equal deleted inserted replaced
258:58424f6abf56 259:11cf20601dec
1 /* Header for code conversion stuff
2 Copyright (C) 1991, 1995 Free Software Foundation, Inc.
3 Copyright (C) 1995 Sun Microsystems, Inc.
4
5 This file is part of XEmacs.
6
7 XEmacs is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by the
9 Free Software Foundation; either version 2, or (at your option) any
10 later version.
11
12 XEmacs is distributed in the hope that it will be useful, but WITHOUT
13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with XEmacs; see the file COPYING. If not, write to
19 the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
20 Boston, MA 02111-1307, USA. */
21
22 /* Synched up with: Mule 2.3. Not in FSF. */
23
24 /* 91.10.09 written by K.Handa <handa@etl.go.jp> */
25 /* Rewritten by Ben Wing <wing@666.com>. */
26
27 #ifndef _XEMACS_MULE_CODING_H_
28 #define _XEMACS_MULE_CODING_H_
29
30 struct decoding_stream;
31 struct encoding_stream;
32
33 /* Coding system types. These go into the TYPE field of a
34 struct Lisp_Coding_System. */
35
36 enum coding_system_type
37 {
38 CODESYS_AUTODETECT, /* Automatic conversion. */
39 #ifdef MULE
40 CODESYS_SHIFT_JIS, /* Shift-JIS; Hankaku (half-width) KANA
41 is also supported. */
42 CODESYS_ISO2022, /* Any ISO2022-compliant coding system.
43 Includes JIS, EUC, CTEXT */
44 CODESYS_BIG5, /* BIG5 (used for Taiwanese). */
45 CODESYS_CCL, /* Converter written in CCL. */
46 #endif
47 CODESYS_NO_CONVERSION /* "No conversion"; used for binary files.
48 We use quotes because there really
49 is some conversion being applied,
50 but it appears to the user as if
51 the text is read in without conversion. */
52 #ifdef DEBUG_XEMACS
53 ,CODESYS_INTERNAL /* Raw (internally-formatted) data. */
54 #endif
55 };
56
57 enum eol_type
58 {
59 EOL_AUTODETECT,
60 EOL_LF,
61 EOL_CRLF,
62 EOL_CR
63 };
64
65 typedef struct charset_conversion_spec charset_conversion_spec;
66 struct charset_conversion_spec
67 {
68 Lisp_Object from_charset;
69 Lisp_Object to_charset;
70 };
71
72 typedef struct
73 {
74 Dynarr_declare (charset_conversion_spec);
75 } charset_conversion_spec_dynarr;
76
77 struct Lisp_Coding_System
78 {
79 struct lcrecord_header header;
80
81 /* Name and doc string of this coding system. */
82 Lisp_Object name, doc_string;
83
84 /* This is the major type of the coding system -- one of Big5, ISO2022,
85 Shift-JIS, etc. See the constants above. */
86 enum coding_system_type type;
87
88 /* Mnemonic string displayed in the modeline when this coding
89 system is active for a particular buffer. */
90 Lisp_Object mnemonic;
91
92 Lisp_Object post_read_conversion, pre_write_conversion;
93
94 enum eol_type eol_type;
95
96 /* Subsidiary coding systems that specify a particular type of EOL
97 marking, rather than autodetecting it. These will only be non-nil
98 if (eol_type == EOL_AUTODETECT). */
99 Lisp_Object eol_lf, eol_crlf, eol_cr;
100
101 struct
102 {
103 /* What are the charsets to be initially designated to G0, G1,
104 G2, G3? If t, no charset is initially designated. If nil,
105 no charset is initially designated and no charset is allowed
106 to be designated. */
107 Lisp_Object initial_charset[4];
108
109 /* If true, a designation escape sequence needs to be sent on output
110 for the charset in G[0-3] before that charset is used. */
111 unsigned char force_charset_on_output[4];
112
113 charset_conversion_spec_dynarr *input_conv;
114 charset_conversion_spec_dynarr *output_conv;
115
116 unsigned int shoort :1; /* C makes you speak Dutch */
117 unsigned int no_ascii_eol :1;
118 unsigned int no_ascii_cntl :1;
119 unsigned int seven :1;
120 unsigned int lock_shift :1;
121 unsigned int no_iso6429 :1;
122 unsigned int escape_quoted :1;
123 } iso2022;
124
125 struct
126 {
127 /* For a CCL coding system, these specify the CCL programs used for
128 decoding (input) and encoding (output). */
129 Lisp_Object decode, encode;
130 } ccl;
131 };
132
133 DECLARE_LRECORD (coding_system, struct Lisp_Coding_System);
134 #define XCODING_SYSTEM(x) XRECORD (x, coding_system, struct Lisp_Coding_System)
135 #define XSETCODING_SYSTEM(x, p) XSETRECORD (x, p, coding_system)
136 #define CODING_SYSTEMP(x) RECORDP (x, coding_system)
137 #define GC_CODING_SYSTEMP(x) GC_RECORDP (x, coding_system)
138 #define CHECK_CODING_SYSTEM(x) CHECK_RECORD (x, coding_system)
139 #define CONCHECK_CODING_SYSTEM(x) CONCHECK_RECORD (x, coding_system)
140
141 #define CODING_SYSTEM_NAME(codesys) ((codesys)->name)
142 #define CODING_SYSTEM_DOC_STRING(codesys) ((codesys)->doc_string)
143 #define CODING_SYSTEM_TYPE(codesys) ((codesys)->type)
144 #define CODING_SYSTEM_MNEMONIC(codesys) ((codesys)->mnemonic)
145 #define CODING_SYSTEM_POST_READ_CONVERSION(codesys) \
146 ((codesys)->post_read_conversion)
147 #define CODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
148 ((codesys)->pre_write_conversion)
149 #define CODING_SYSTEM_EOL_TYPE(codesys) ((codesys)->eol_type)
150 #define CODING_SYSTEM_EOL_LF(codesys) ((codesys)->eol_lf)
151 #define CODING_SYSTEM_EOL_CRLF(codesys) ((codesys)->eol_crlf)
152 #define CODING_SYSTEM_EOL_CR(codesys) ((codesys)->eol_cr)
153 #define CODING_SYSTEM_ISO2022_INITIAL_CHARSET(codesys, g) \
154 ((codesys)->iso2022.initial_charset[g])
155 #define CODING_SYSTEM_ISO2022_FORCE_CHARSET_ON_OUTPUT(codesys, g) \
156 ((codesys)->iso2022.force_charset_on_output[g])
157 #define CODING_SYSTEM_ISO2022_SHORT(codesys) ((codesys)->iso2022.shoort)
158 #define CODING_SYSTEM_ISO2022_NO_ASCII_EOL(codesys) \
159 ((codesys)->iso2022.no_ascii_eol)
160 #define CODING_SYSTEM_ISO2022_NO_ASCII_CNTL(codesys) \
161 ((codesys)->iso2022.no_ascii_cntl)
162 #define CODING_SYSTEM_ISO2022_SEVEN(codesys) ((codesys)->iso2022.seven)
163 #define CODING_SYSTEM_ISO2022_LOCK_SHIFT(codesys) \
164 ((codesys)->iso2022.lock_shift)
165 #define CODING_SYSTEM_ISO2022_NO_ISO6429(codesys) \
166 ((codesys)->iso2022.no_iso6429)
167 #define CODING_SYSTEM_ISO2022_ESCAPE_QUOTED(codesys) \
168 ((codesys)->iso2022.escape_quoted)
169 #define CODING_SYSTEM_CCL_DECODE(codesys) ((codesys)->ccl.decode)
170 #define CODING_SYSTEM_CCL_ENCODE(codesys) ((codesys)->ccl.encode)
171
172 #define XCODING_SYSTEM_NAME(codesys) \
173 CODING_SYSTEM_NAME (XCODING_SYSTEM (codesys))
174 #define XCODING_SYSTEM_DOC_STRING(codesys) \
175 CODING_SYSTEM_DOC_STRING (XCODING_SYSTEM (codesys))
176 #define XCODING_SYSTEM_TYPE(codesys) \
177 CODING_SYSTEM_TYPE (XCODING_SYSTEM (codesys))
178 #define XCODING_SYSTEM_MNEMONIC(codesys) \
179 CODING_SYSTEM_MNEMONIC (XCODING_SYSTEM (codesys))
180 #define XCODING_SYSTEM_POST_READ_CONVERSION(codesys) \
181 CODING_SYSTEM_POST_READ_CONVERSION (XCODING_SYSTEM (codesys))
182 #define XCODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
183 CODING_SYSTEM_PRE_WRITE_CONVERSION (XCODING_SYSTEM (codesys))
184 #define XCODING_SYSTEM_EOL_TYPE(codesys) \
185 CODING_SYSTEM_EOL_TYPE (XCODING_SYSTEM (codesys))
186 #define XCODING_SYSTEM_EOL_LF(codesys) \
187 CODING_SYSTEM_EOL_LF (XCODING_SYSTEM (codesys))
188 #define XCODING_SYSTEM_EOL_CRLF(codesys) \
189 CODING_SYSTEM_EOL_CRLF (XCODING_SYSTEM (codesys))
190 #define XCODING_SYSTEM_EOL_CR(codesys) \
191 CODING_SYSTEM_EOL_CR (XCODING_SYSTEM (codesys))
192 #define XCODING_SYSTEM_ISO2022_INITIAL_CHARSET(codesys, g) \
193 CODING_SYSTEM_ISO2022_INITIAL_CHARSET (XCODING_SYSTEM (codesys), g)
194 #define XCODING_SYSTEM_ISO2022_FORCE_CHARSET_ON_OUTPUT(codesys, g) \
195 CODING_SYSTEM_ISO2022_FORCE_CHARSET_ON_OUTPUT (XCODING_SYSTEM (codesys), g)
196 #define XCODING_SYSTEM_ISO2022_SHORT(codesys) \
197 CODING_SYSTEM_ISO2022_SHORT (XCODING_SYSTEM (codesys))
198 #define XCODING_SYSTEM_ISO2022_NO_ASCII_EOL(codesys) \
199 CODING_SYSTEM_ISO2022_NO_ASCII_EOL (XCODING_SYSTEM (codesys))
200 #define XCODING_SYSTEM_ISO2022_NO_ASCII_CNTL(codesys) \
201 CODING_SYSTEM_ISO2022_NO_ASCII_CNTL (XCODING_SYSTEM (codesys))
202 #define XCODING_SYSTEM_ISO2022_SEVEN(codesys) \
203 CODING_SYSTEM_ISO2022_SEVEN (XCODING_SYSTEM (codesys))
204 #define XCODING_SYSTEM_ISO2022_LOCK_SHIFT(codesys) \
205 CODING_SYSTEM_ISO2022_LOCK_SHIFT (XCODING_SYSTEM (codesys))
206 #define XCODING_SYSTEM_ISO2022_NO_ISO6429(codesys) \
207 CODING_SYSTEM_ISO2022_NO_ISO6429 (XCODING_SYSTEM (codesys))
208 #define XCODING_SYSTEM_ISO2022_ESCAPE_QUOTED(codesys) \
209 CODING_SYSTEM_ISO2022_ESCAPE_QUOTED (XCODING_SYSTEM (codesys))
210 #define XCODING_SYSTEM_CCL_DECODE(codesys) \
211 CODING_SYSTEM_CCL_DECODE (XCODING_SYSTEM (codesys))
212 #define XCODING_SYSTEM_CCL_ENCODE(codesys) \
213 CODING_SYSTEM_CCL_ENCODE (XCODING_SYSTEM (codesys))
214
215 extern Lisp_Object Qbuffer_file_coding_system, Qcoding_system_error;
216
217 extern Lisp_Object Vkeyboard_coding_system;
218 extern Lisp_Object Vterminal_coding_system;
219 extern Lisp_Object Vcoding_system_for_read;
220 extern Lisp_Object Vcoding_system_for_write;
221 extern Lisp_Object Vpathname_coding_system;
222
223 extern Lisp_Object Qescape_quoted;
224
225 /* Flags indicating current state while converting code. */
226
227 /* Used by everyone. */
228
229 #define CODING_STATE_END (1 << 0) /* If set, this is the last chunk of
230 data being processed. When this
231 is finished, output any necessary
232 terminating control characters,
233 escape sequences, etc. */
234 #define CODING_STATE_CR (1 << 1) /* If set, we just saw a CR. */
235
236
237 /* Used by Big 5 on output. */
238
239 #define CODING_STATE_BIG5_1 (1 << 2) /* If set, we just encountered
240 LEADING_BYTE_BIG5_1. */
241 #define CODING_STATE_BIG5_2 (1 << 3) /* If set, we just encountered
242 LEADING_BYTE_BIG5_2. */
243
244
245 /* Used by ISO2022 on input and output. */
246
247 #define CODING_STATE_R2L (1 << 4) /* If set, the current
248 directionality is right-to-left.
249 Otherwise, it's left-to-right. */
250
251
252 /* Used by ISO2022 on input. */
253
254 #define CODING_STATE_ESCAPE (1 << 5) /* If set, we're currently parsing
255 an escape sequence and the upper
256 16 bits should be looked at to
257 indicate what partial escape
258 sequence we've seen so far.
259 Otherwise, we're running
260 through actual text. */
261 #define CODING_STATE_SS2 (1 << 6) /* If set, G2 is invoked into GL, but
262 only for the next character. */
263 #define CODING_STATE_SS3 (1 << 7) /* If set, G3 is invoked into GL,
264 but only for the next character.
265 If both CODING_STATE_SS2 and
266 CODING_STATE_SS3 are set,
267 CODING_STATE_SS2 overrides; but
268 this probably indicates an error
269 in the text encoding. */
270 #define CODING_STATE_COMPOSITE (1 << 8) /* If set, we're currently processing
271 a composite character (i.e. a
272 character constructed by
273 overstriking two or more
274 characters). */
275
276
277 /* CODING_STATE_ISO2022_LOCK is the mask of flags that remain on until
278 explicitly turned off when in the ISO2022 encoder/decoder. Other flags are
279 turned off at the end of processing each character or escape sequence. */
280 # define CODING_STATE_ISO2022_LOCK \
281 (CODING_STATE_END | CODING_STATE_COMPOSITE | CODING_STATE_R2L)
282 #define CODING_STATE_BIG5_LOCK \
283 CODING_STATE_END
284
285 /* Flags indicating what we've seen so far when parsing an
286 ISO2022 escape sequence. */
287 enum iso_esc_flag
288 {
289 /* Partial sequences */
290 ISO_ESC_NOTHING, /* Nothing has been seen. */
291 ISO_ESC, /* We've seen ESC. */
292 ISO_ESC_2_4, /* We've seen ESC $. This indicates
293 that we're designating a multi-byte, rather
294 than a single-byte, character set. */
295 ISO_ESC_2_8, /* We've seen ESC 0x28, i.e. ESC (.
296 This means designate a 94-character
297 character set into G0. */
298 ISO_ESC_2_9, /* We've seen ESC 0x29 -- designate a
299 94-character character set into G1. */
300 ISO_ESC_2_10, /* We've seen ESC 0x2A. */
301 ISO_ESC_2_11, /* We've seen ESC 0x2B. */
302 ISO_ESC_2_12, /* We've seen ESC 0x2C -- designate a
303 96-character character set into G0.
304 (This is not ISO2022-standard.
305 The following 96-character
306 control sequences are standard,
307 though.) */
308 ISO_ESC_2_13, /* We've seen ESC 0x2D -- designate a
309 96-character character set into G1.
310 */
311 ISO_ESC_2_14, /* We've seen ESC 0x2E. */
312 ISO_ESC_2_15, /* We've seen ESC 0x2F. */
313 ISO_ESC_2_4_8, /* We've seen ESC $ 0x28 -- designate
314 a 94^N character set into G0. */
315 ISO_ESC_2_4_9, /* We've seen ESC $ 0x29. */
316 ISO_ESC_2_4_10, /* We've seen ESC $ 0x2A. */
317 ISO_ESC_2_4_11, /* We've seen ESC $ 0x2B. */
318 ISO_ESC_2_4_12, /* We've seen ESC $ 0x2C. */
319 ISO_ESC_2_4_13, /* We've seen ESC $ 0x2D. */
320 ISO_ESC_2_4_14, /* We've seen ESC $ 0x2E. */
321 ISO_ESC_2_4_15, /* We've seen ESC $ 0x2F. */
322 ISO_ESC_5_11, /* We've seen ESC [ or 0x9B. This
323 starts a directionality-control
324 sequence. The next character
325 must be 0, 1, 2, or ]. */
326 ISO_ESC_5_11_0, /* We've seen 0x9B 0. The next
327 character must be ]. */
328 ISO_ESC_5_11_1, /* We've seen 0x9B 1. The next
329 character must be ]. */
330 ISO_ESC_5_11_2, /* We've seen 0x9B 2. The next
331 character must be ]. */
332
333 /* Full sequences. */
334 ISO_ESC_START_COMPOSITE, /* Private usage for START COMPOSING */
335 ISO_ESC_END_COMPOSITE, /* Private usage for END COMPOSING */
336 ISO_ESC_SINGLE_SHIFT, /* We've seen a complete single-shift sequence. */
337 ISO_ESC_LOCKING_SHIFT,/* We've seen a complete locking-shift sequence. */
338 ISO_ESC_DESIGNATE, /* We've seen a complete designation sequence. */
339 ISO_ESC_DIRECTIONALITY,/* We've seen a complete ISO6429 directionality
340 sequence. */
341 ISO_ESC_LITERAL /* We've seen a literal character ala
342 escape-quoting. */
343 };
344
345 /* Macros to define code of control characters for ISO2022's functions. */
346 /* code */ /* function */
347 #define ISO_CODE_LF 0x0A /* line-feed */
348 #define ISO_CODE_CR 0x0D /* carriage-return */
349 #define ISO_CODE_SO 0x0E /* shift-out */
350 #define ISO_CODE_SI 0x0F /* shift-in */
351 #define ISO_CODE_ESC 0x1B /* escape */
352 #define ISO_CODE_DEL 0x7F /* delete */
353 #define ISO_CODE_SS2 0x8E /* single-shift-2 */
354 #define ISO_CODE_SS3 0x8F /* single-shift-3 */
355 #define ISO_CODE_CSI 0x9B /* control-sequence-introduce */
356
357 /* Macros to access an encoding stream or decoding stream */
358
359 #define CODING_STREAM_DECOMPOSE(str, flags, ch) \
360 do { \
361 flags = (str)->flags; \
362 ch = (str)->ch; \
363 } while (0)
364
365 #define CODING_STREAM_COMPOSE(str, flags, ch) \
366 do { \
367 (str)->flags = flags; \
368 (str)->ch = ch; \
369 } while (0)
370
371
372 /* For detecting the encoding of text */
373 enum coding_category_type
374 {
375 CODING_CATEGORY_SHIFT_JIS,
376 CODING_CATEGORY_ISO_7, /* ISO2022 system using only seven-bit bytes,
377 no locking shift */
378 CODING_CATEGORY_ISO_8_DESIGNATE, /* ISO2022 system using eight-bit bytes,
379 no locking shift, no single shift,
380 using designation to switch charsets */
381 CODING_CATEGORY_ISO_8_1, /* ISO2022 system using eight-bit bytes,
382 no locking shift, no designation sequences,
383 one-dimension characters in the upper half. */
384 CODING_CATEGORY_ISO_8_2, /* ISO2022 system using eight-bit bytes,
385 no locking shift, no designation sequences,
386 two-dimension characters in the upper half. */
387 CODING_CATEGORY_ISO_LOCK_SHIFT, /* ISO2022 system using locking shift */
388 CODING_CATEGORY_BIG5,
389 CODING_CATEGORY_NO_CONVERSION
390 };
391
392 #define CODING_CATEGORY_LAST CODING_CATEGORY_NO_CONVERSION
393
394 #define CODING_CATEGORY_SHIFT_JIS_MASK \
395 (1 << CODING_CATEGORY_SHIFT_JIS)
396 #define CODING_CATEGORY_ISO_7_MASK \
397 (1 << CODING_CATEGORY_ISO_7)
398 #define CODING_CATEGORY_ISO_8_DESIGNATE_MASK \
399 (1 << CODING_CATEGORY_ISO_8_DESIGNATE)
400 #define CODING_CATEGORY_ISO_8_1_MASK \
401 (1 << CODING_CATEGORY_ISO_8_1)
402 #define CODING_CATEGORY_ISO_8_2_MASK \
403 (1 << CODING_CATEGORY_ISO_8_2)
404 #define CODING_CATEGORY_ISO_LOCK_SHIFT_MASK \
405 (1 << CODING_CATEGORY_ISO_LOCK_SHIFT)
406 #define CODING_CATEGORY_BIG5_MASK \
407 (1 << CODING_CATEGORY_BIG5)
408 #define CODING_CATEGORY_NO_CONVERSION_MASK \
409 (1 << CODING_CATEGORY_NO_CONVERSION)
410 #define CODING_CATEGORY_NOT_FINISHED_MASK \
411 (1 << 30)
412
413 /* Convert shift-JIS code (sj1, sj2) into internal string
414 representation (c1, c2). (The leading byte is assumed.) */
415
416 #define DECODE_SJIS(sj1, sj2, c1, c2) \
417 do { \
418 int I1 = sj1, I2 = sj2; \
419 if (I2 >= 0x9f) \
420 c1 = (I1 << 1) - ((I1 >= 0xe0) ? 0xe0 : 0x60), \
421 c2 = I2 + 2; \
422 else \
423 c1 = (I1 << 1) - ((I1 >= 0xe0) ? 0xe1 : 0x61), \
424 c2 = I2 + ((I2 >= 0x7f) ? 0x60 : 0x61); \
425 } while (0)
426
427 /* Convert the internal string representation of a Shift-JIS character
428 (c1, c2) into Shift-JIS code (sj1, sj2). The leading byte is
429 assumed. */
430
431 #define ENCODE_SJIS(c1, c2, sj1, sj2) \
432 do { \
433 int I1 = c1, I2 = c2; \
434 if (I1 & 1) \
435 sj1 = (I1 >> 1) + ((I1 < 0xdf) ? 0x31 : 0x71), \
436 sj2 = I2 - ((I2 >= 0xe0) ? 0x60 : 0x61); \
437 else \
438 sj1 = (I1 >> 1) + ((I1 < 0xdf) ? 0x30 : 0x70), \
439 sj2 = I2 - 2; \
440 } while (0)
441
442 extern Lisp_Object make_decoding_input_stream (Lstream *stream,
443 Lisp_Object codesys);
444 extern Lisp_Object make_encoding_input_stream (Lstream *stream,
445 Lisp_Object codesys);
446 extern Lisp_Object make_decoding_output_stream (Lstream *stream,
447 Lisp_Object codesys);
448 extern Lisp_Object make_encoding_output_stream (Lstream *stream,
449 Lisp_Object codesys);
450 extern Lisp_Object decoding_stream_coding_system (Lstream *stream);
451 extern Lisp_Object encoding_stream_coding_system (Lstream *stream);
452 extern void set_decoding_stream_coding_system (Lstream *stream,
453 Lisp_Object codesys);
454 extern void set_encoding_stream_coding_system (Lstream *stream,
455 Lisp_Object codesys);
456 extern void determine_real_coding_system (Lstream *stream, Lisp_Object *codesys_in_out,
457 enum eol_type *eol_type_in_out);
458
459
460 #ifndef MULE
461 #define MIN_LEADING_BYTE 0x80
462 /* These need special treatment in a string and/or character */
463 #define LEADING_BYTE_COMPOSITE 0x80 /* for a composite character */
464 #define LEADING_BYTE_CONTROL_1 0x8F /* represent normal 80-9F */
465 #define LEADING_BYTE_LATIN_ISO8859_1 0x81 /* Right half of ISO 8859-1 */
466 #define BYTE_C1_P(c) ((unsigned int) ((unsigned int) (c) - 0x80) < 0x20)
467 #define BUFBYTE_FIRST_BYTE_P(c) ((c) < 0xA0)
468 #define BUFBYTE_LEADING_BYTE_P(c) BYTE_C1_P (c)
469 #endif /* not MULE */
470 #endif /* _XEMACS_MULE_CODING_H_ */
471