Mercurial > hg > xemacs-beta

comparison src/mule-coding.h @ 70:131b0175ea99 r20-0b30

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Import from CVS: tag r20-0b30
author cvs
date Mon, 13 Aug 2007 09:02:59 +0200
parents
children fe104dbd9147
comparison
equal deleted inserted replaced
69:804d1389bcd6 70:131b0175ea99
1 /* Header for code conversion stuff
2 Copyright (C) 1991, 1995 Free Software Foundation, Inc.
3 Copyright (C) 1995 Sun Microsystems, Inc.
4
5 This file is part of XEmacs.
6
7 XEmacs is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by the
9 Free Software Foundation; either version 2, or (at your option) any
10 later version.
11
12 XEmacs is distributed in the hope that it will be useful, but WITHOUT
13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with XEmacs; see the file COPYING. If not, write to
19 the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
20 Boston, MA 02111-1307, USA. */
21
22 /* Synched up with: Mule 2.3. Not in FSF. */
23
24 /* 91.10.09 written by K.Handa <handa@etl.go.jp> */
25 /* Rewritten by Ben Wing <wing@666.com>. */
26
27 #ifndef _XEMACS_MULE_CODING_H_
28 #define _XEMACS_MULE_CODING_H_
29
30 struct decoding_stream;
31 struct encoding_stream;
32
33 /* Coding system types. These go into the TYPE field of a
34 struct Lisp_Coding_System. */
35
36 enum coding_system_type
37 {
38 CODESYS_AUTODETECT, /* Automatic conversion. */
39 CODESYS_SHIFT_JIS, /* Shift-JIS; Hankaku (half-width) KANA
40 is also supported. */
41 CODESYS_ISO2022, /* Any ISO2022-compliant coding system.
42 Includes JIS, EUC, CTEXT */
43 CODESYS_BIG5, /* BIG5 (used for Taiwanese). */
44 CODESYS_CCL, /* Converter written in CCL. */
45 CODESYS_NO_CONVERSION /* "No conversion"; used for binary files.
46 We use quotes because there really
47 is some conversion being applied,
48 but it appears to the user as if
49 the text is read in without conversion. */
50 #ifdef DEBUG_XEMACS
51 ,CODESYS_INTERNAL /* Raw (internally-formatted) data. */
52 #endif
53 };
54
55 enum eol_type
56 {
57 EOL_AUTODETECT,
58 EOL_LF,
59 EOL_CRLF,
60 EOL_CR
61 };
62
63 /* This holds the current state of a running CCL program. */
64 struct ccl_program
65 {
66 Lisp_Object saved_vector;
67 Lisp_Object *prog; /* compiled code */
68 int size; /* size of compiled code */
69 int ic; /* instruction counter */
70 int reg[8]; /* reg[7] is used for `condition' */
71 int end_flag; /* set when processing the last block */
72 int status;
73 };
74
75 struct charset_conversion_spec
76 {
77 Lisp_Object from_charset;
78 Lisp_Object to_charset;
79 };
80
81 typedef struct charset_conversion_spec_dynarr_type
82 {
83 Dynarr_declare (struct charset_conversion_spec);
84 } charset_conversion_spec_dynarr;
85
86 struct Lisp_Coding_System
87 {
88 struct lcrecord_header header;
89
90 /* Name and doc string of this coding system. */
91 Lisp_Object name, doc_string;
92
93 /* This is the major type of the coding system -- one of Big5, ISO2022,
94 Shift-JIS, etc. See the constants above. */
95 enum coding_system_type type;
96
97 /* Mnemonic string displayed in the modeline when this coding
98 system is active for a particular buffer. */
99 Lisp_Object mnemonic;
100
101 Lisp_Object post_read_conversion, pre_write_conversion;
102
103 enum eol_type eol_type;
104
105 /* Subsidiary coding systems that specify a particular type of EOL
106 marking, rather than autodetecting it. These will only be non-nil
107 if (eol_type == EOL_AUTODETECT). */
108 Lisp_Object eol_lf, eol_crlf, eol_cr;
109
110 struct
111 {
112 /* What are the charsets to be initially designated to G0, G1,
113 G2, G3? If t, no charset is initially designated. If nil,
114 no charset is initially designated and no charset is allowed
115 to be designated. */
116 Lisp_Object initial_charset[4];
117
118 /* If true, a designation escape sequence needs to be sent on output
119 for the charset in G[0-3] before that charset is used. */
120 unsigned char force_charset_on_output[4];
121
122 charset_conversion_spec_dynarr *input_conv;
123 charset_conversion_spec_dynarr *output_conv;
124
125 unsigned int shoort :1; /* C makes you speak Dutch */
126 unsigned int no_ascii_eol :1;
127 unsigned int no_ascii_cntl :1;
128 unsigned int seven :1;
129 unsigned int lock_shift :1;
130 unsigned int no_iso6429 :1;
131 unsigned int escape_quoted :1;
132 } iso2022;
133
134 struct
135 {
136 /* For a CCL coding system, these specify the CCL programs used for
137 decoding (input) and encoding (output). */
138 Lisp_Object decode, encode;
139 } ccl;
140 };
141
142 DECLARE_LRECORD (coding_system, struct Lisp_Coding_System);
143 #define XCODING_SYSTEM(x) XRECORD (x, coding_system, struct Lisp_Coding_System)
144 #define XSETCODING_SYSTEM(x, p) XSETRECORD (x, p, coding_system)
145 #define CODING_SYSTEMP(x) RECORDP (x, coding_system)
146 #define GC_CODING_SYSTEMP(x) GC_RECORDP (x, coding_system)
147 #define CHECK_CODING_SYSTEM(x) CHECK_RECORD (x, coding_system)
148 #define CONCHECK_CODING_SYSTEM(x) CONCHECK_RECORD (x, coding_system)
149
150 #define CODING_SYSTEM_NAME(codesys) ((codesys)->name)
151 #define CODING_SYSTEM_DOC_STRING(codesys) ((codesys)->doc_string)
152 #define CODING_SYSTEM_TYPE(codesys) ((codesys)->type)
153 #define CODING_SYSTEM_MNEMONIC(codesys) ((codesys)->mnemonic)
154 #define CODING_SYSTEM_POST_READ_CONVERSION(codesys) \
155 ((codesys)->post_read_conversion)
156 #define CODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
157 ((codesys)->pre_write_conversion)
158 #define CODING_SYSTEM_EOL_TYPE(codesys) ((codesys)->eol_type)
159 #define CODING_SYSTEM_EOL_LF(codesys) ((codesys)->eol_lf)
160 #define CODING_SYSTEM_EOL_CRLF(codesys) ((codesys)->eol_crlf)
161 #define CODING_SYSTEM_EOL_CR(codesys) ((codesys)->eol_cr)
162 #define CODING_SYSTEM_ISO2022_INITIAL_CHARSET(codesys, g) \
163 ((codesys)->iso2022.initial_charset[g])
164 #define CODING_SYSTEM_ISO2022_FORCE_CHARSET_ON_OUTPUT(codesys, g) \
165 ((codesys)->iso2022.force_charset_on_output[g])
166 #define CODING_SYSTEM_ISO2022_SHORT(codesys) ((codesys)->iso2022.shoort)
167 #define CODING_SYSTEM_ISO2022_NO_ASCII_EOL(codesys) \
168 ((codesys)->iso2022.no_ascii_eol)
169 #define CODING_SYSTEM_ISO2022_NO_ASCII_CNTL(codesys) \
170 ((codesys)->iso2022.no_ascii_cntl)
171 #define CODING_SYSTEM_ISO2022_SEVEN(codesys) ((codesys)->iso2022.seven)
172 #define CODING_SYSTEM_ISO2022_LOCK_SHIFT(codesys) \
173 ((codesys)->iso2022.lock_shift)
174 #define CODING_SYSTEM_ISO2022_NO_ISO6429(codesys) \
175 ((codesys)->iso2022.no_iso6429)
176 #define CODING_SYSTEM_ISO2022_ESCAPE_QUOTED(codesys) \
177 ((codesys)->iso2022.escape_quoted)
178 #define CODING_SYSTEM_CCL_DECODE(codesys) ((codesys)->ccl.decode)
179 #define CODING_SYSTEM_CCL_ENCODE(codesys) ((codesys)->ccl.encode)
180
181 #define XCODING_SYSTEM_NAME(codesys) \
182 CODING_SYSTEM_NAME (XCODING_SYSTEM (codesys))
183 #define XCODING_SYSTEM_DOC_STRING(codesys) \
184 CODING_SYSTEM_DOC_STRING (XCODING_SYSTEM (codesys))
185 #define XCODING_SYSTEM_TYPE(codesys) \
186 CODING_SYSTEM_TYPE (XCODING_SYSTEM (codesys))
187 #define XCODING_SYSTEM_MNEMONIC(codesys) \
188 CODING_SYSTEM_MNEMONIC (XCODING_SYSTEM (codesys))
189 #define XCODING_SYSTEM_POST_READ_CONVERSION(codesys) \
190 CODING_SYSTEM_POST_READ_CONVERSION (XCODING_SYSTEM (codesys))
191 #define XCODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
192 CODING_SYSTEM_PRE_WRITE_CONVERSION (XCODING_SYSTEM (codesys))
193 #define XCODING_SYSTEM_EOL_TYPE(codesys) \
194 CODING_SYSTEM_EOL_TYPE (XCODING_SYSTEM (codesys))
195 #define XCODING_SYSTEM_EOL_LF(codesys) \
196 CODING_SYSTEM_EOL_LF (XCODING_SYSTEM (codesys))
197 #define XCODING_SYSTEM_EOL_CRLF(codesys) \
198 CODING_SYSTEM_EOL_CRLF (XCODING_SYSTEM (codesys))
199 #define XCODING_SYSTEM_EOL_CR(codesys) \
200 CODING_SYSTEM_EOL_CR (XCODING_SYSTEM (codesys))
201 #define XCODING_SYSTEM_ISO2022_INITIAL_CHARSET(codesys, g) \
202 CODING_SYSTEM_ISO2022_INITIAL_CHARSET (XCODING_SYSTEM (codesys), g)
203 #define XCODING_SYSTEM_ISO2022_FORCE_CHARSET_ON_OUTPUT(codesys, g) \
204 CODING_SYSTEM_ISO2022_FORCE_CHARSET_ON_OUTPUT (XCODING_SYSTEM (codesys), g)
205 #define XCODING_SYSTEM_ISO2022_SHORT(codesys) \
206 CODING_SYSTEM_ISO2022_SHORT (XCODING_SYSTEM (codesys))
207 #define XCODING_SYSTEM_ISO2022_NO_ASCII_EOL(codesys) \
208 CODING_SYSTEM_ISO2022_NO_ASCII_EOL (XCODING_SYSTEM (codesys))
209 #define XCODING_SYSTEM_ISO2022_NO_ASCII_CNTL(codesys) \
210 CODING_SYSTEM_ISO2022_NO_ASCII_CNTL (XCODING_SYSTEM (codesys))
211 #define XCODING_SYSTEM_ISO2022_SEVEN(codesys) \
212 CODING_SYSTEM_ISO2022_SEVEN (XCODING_SYSTEM (codesys))
213 #define XCODING_SYSTEM_ISO2022_LOCK_SHIFT(codesys) \
214 CODING_SYSTEM_ISO2022_LOCK_SHIFT (XCODING_SYSTEM (codesys))
215 #define XCODING_SYSTEM_ISO2022_NO_ISO6429(codesys) \
216 CODING_SYSTEM_ISO2022_NO_ISO6429 (XCODING_SYSTEM (codesys))
217 #define XCODING_SYSTEM_ISO2022_ESCAPE_QUOTED(codesys) \
218 CODING_SYSTEM_ISO2022_ESCAPE_QUOTED (XCODING_SYSTEM (codesys))
219 #define XCODING_SYSTEM_CCL_DECODE(codesys) \
220 CODING_SYSTEM_CCL_DECODE (XCODING_SYSTEM (codesys))
221 #define XCODING_SYSTEM_CCL_ENCODE(codesys) \
222 CODING_SYSTEM_CCL_ENCODE (XCODING_SYSTEM (codesys))
223
224 extern Lisp_Object Qfile_coding_system, Qcoding_system_error;
225
226 extern Lisp_Object Vkeyboard_coding_system;
227 extern Lisp_Object Vterminal_coding_system;
228 extern Lisp_Object Vprocess_input_coding_system;
229 extern Lisp_Object Vprocess_output_coding_system;
230 extern Lisp_Object Vpathname_coding_system;
231
232 extern Lisp_Object Qescape_quoted;
233
234 /* Flags indicating current state while converting code. */
235
236 /* Used by everyone. */
237
238 #define CODING_STATE_END (1 << 0) /* If set, this is the last chunk of
239 data being processed. When this
240 is finished, output any necessary
241 terminating control characters,
242 escape sequences, etc. */
243 #define CODING_STATE_CR (1 << 1) /* If set, we just saw a CR. */
244
245
246 /* Used by Big 5 on output. */
247
248 #define CODING_STATE_BIG5_1 (1 << 2) /* If set, we just encountered
249 LEADING_BYTE_BIG5_1. */
250 #define CODING_STATE_BIG5_2 (1 << 3) /* If set, we just encountered
251 LEADING_BYTE_BIG5_2. */
252
253
254 /* Used by ISO2022 on input and output. */
255
256 #define CODING_STATE_R2L (1 << 4) /* If set, the current
257 directionality is right-to-left.
258 Otherwise, it's left-to-right. */
259
260
261 /* Used by ISO2022 on input. */
262
263 #define CODING_STATE_ESCAPE (1 << 5) /* If set, we're currently parsing
264 an escape sequence and the upper
265 16 bits should be looked at to
266 indicate what partial escape
267 sequence we've seen so far.
268 Otherwise, we're running
269 through actual text. */
270 #define CODING_STATE_SS2 (1 << 6) /* If set, G2 is invoked into GL, but
271 only for the next character. */
272 #define CODING_STATE_SS3 (1 << 7) /* If set, G3 is invoked into GL,
273 but only for the next character.
274 If both CODING_STATE_SS2 and
275 CODING_STATE_SS3 are set,
276 CODING_STATE_SS2 overrides; but
277 this probably indicates an error
278 in the text encoding. */
279 #define CODING_STATE_COMPOSITE (1 << 8) /* If set, we're currently processing
280 a composite character (i.e. a
281 character constructed by
282 overstriking two or more
283 characters). */
284
285
286 /* CODING_STATE_ISO2022_LOCK is the mask of flags that remain on until
287 explicitly turned off when in the ISO2022 encoder/decoder. Other flags are
288 turned off at the end of processing each character or escape sequence. */
289 # define CODING_STATE_ISO2022_LOCK \
290 (CODING_STATE_END | CODING_STATE_COMPOSITE | CODING_STATE_R2L)
291 #define CODING_STATE_BIG5_LOCK \
292 CODING_STATE_END
293
294 /* Flags indicating what we've seen so far when parsing an
295 ISO2022 escape sequence. */
296 enum iso_esc_flag
297 {
298 /* Partial sequences */
299 ISO_ESC_NOTHING, /* Nothing has been seen. */
300 ISO_ESC, /* We've seen ESC. */
301 ISO_ESC_2_4, /* We've seen ESC $. This indicates
302 that we're designating a multi-byte, rather
303 than a single-byte, character set. */
304 ISO_ESC_2_8, /* We've seen ESC 0x28, i.e. ESC (.
305 This means designate a 94-character
306 character set into G0. */
307 ISO_ESC_2_9, /* We've seen ESC 0x29 -- designate a
308 94-character character set into G1. */
309 ISO_ESC_2_10, /* We've seen ESC 0x2A. */
310 ISO_ESC_2_11, /* We've seen ESC 0x2B. */
311 ISO_ESC_2_12, /* We've seen ESC 0x2C -- designate a
312 96-character character set into G0.
313 (This is not ISO2022-standard.
314 The following 96-character
315 control sequences are standard,
316 though.) */
317 ISO_ESC_2_13, /* We've seen ESC 0x2D -- designate a
318 96-character character set into G1.
319 */
320 ISO_ESC_2_14, /* We've seen ESC 0x2E. */
321 ISO_ESC_2_15, /* We've seen ESC 0x2F. */
322 ISO_ESC_2_4_8, /* We've seen ESC $ 0x28 -- designate
323 a 94^N character set into G0. */
324 ISO_ESC_2_4_9, /* We've seen ESC $ 0x29. */
325 ISO_ESC_2_4_10, /* We've seen ESC $ 0x2A. */
326 ISO_ESC_2_4_11, /* We've seen ESC $ 0x2B. */
327 ISO_ESC_2_4_12, /* We've seen ESC $ 0x2C. */
328 ISO_ESC_2_4_13, /* We've seen ESC $ 0x2D. */
329 ISO_ESC_2_4_14, /* We've seen ESC $ 0x2E. */
330 ISO_ESC_2_4_15, /* We've seen ESC $ 0x2F. */
331 ISO_ESC_5_11, /* We've seen ESC [ or 0x9B. This
332 starts a directionality-control
333 sequence. The next character
334 must be 0, 1, 2, or ]. */
335 ISO_ESC_5_11_0, /* We've seen 0x9B 0. The next
336 character must be ]. */
337 ISO_ESC_5_11_1, /* We've seen 0x9B 1. The next
338 character must be ]. */
339 ISO_ESC_5_11_2, /* We've seen 0x9B 2. The next
340 character must be ]. */
341
342 /* Full sequences. */
343 ISO_ESC_START_COMPOSITE, /* Private usage for START COMPOSING */
344 ISO_ESC_END_COMPOSITE, /* Private usage for END COMPOSING */
345 ISO_ESC_SINGLE_SHIFT, /* We've seen a complete single-shift sequence. */
346 ISO_ESC_LOCKING_SHIFT,/* We've seen a complete locking-shift sequence. */
347 ISO_ESC_DESIGNATE, /* We've seen a complete designation sequence. */
348 ISO_ESC_DIRECTIONALITY,/* We've seen a complete ISO6429 directionality
349 sequence. */
350 ISO_ESC_LITERAL /* We've seen a literal character ala
351 escape-quoting. */
352 };
353
354 /* Macros to define code of control characters for ISO2022's functions. */
355 /* code */ /* function */
356 #define ISO_CODE_LF 0x0A /* line-feed */
357 #define ISO_CODE_CR 0x0D /* carriage-return */
358 #define ISO_CODE_SO 0x0E /* shift-out */
359 #define ISO_CODE_SI 0x0F /* shift-in */
360 #define ISO_CODE_ESC 0x1B /* escape */
361 #define ISO_CODE_DEL 0x7F /* delete */
362 #define ISO_CODE_SS2 0x8E /* single-shift-2 */
363 #define ISO_CODE_SS3 0x8F /* single-shift-3 */
364 #define ISO_CODE_CSI 0x9B /* control-sequence-introduce */
365
366 /* Macros to access an encoding stream or decoding stream */
367
368 #define CODING_STREAM_DECOMPOSE(str, flags, ch) \
369 do { \
370 flags = (str)->flags; \
371 ch = (str)->ch; \
372 } while (0)
373
374 #define CODING_STREAM_COMPOSE(str, flags, ch) \
375 do { \
376 (str)->flags = flags; \
377 (str)->ch = ch; \
378 } while (0)
379
380
381 /* For detecting the encoding of text */
382 enum coding_category_type
383 {
384 CODING_CATEGORY_SHIFT_JIS,
385 CODING_CATEGORY_ISO_7, /* ISO2022 system using only seven-bit bytes,
386 no locking shift */
387 CODING_CATEGORY_ISO_8_DESIGNATE, /* ISO2022 system using eight-bit bytes,
388 no locking shift, no single shift,
389 using designation to switch charsets */
390 CODING_CATEGORY_ISO_8_1, /* ISO2022 system using eight-bit bytes,
391 no locking shift, no designation sequences,
392 one-dimension characters in the upper half. */
393 CODING_CATEGORY_ISO_8_2, /* ISO2022 system using eight-bit bytes,
394 no locking shift, no designation sequences,
395 two-dimension characters in the upper half. */
396 CODING_CATEGORY_ISO_LOCK_SHIFT, /* ISO2022 system using locking shift */
397 CODING_CATEGORY_BIG5,
398 CODING_CATEGORY_NO_CONVERSION
399 };
400
401 #define CODING_CATEGORY_LAST CODING_CATEGORY_NO_CONVERSION
402
403 #define CODING_CATEGORY_SHIFT_JIS_MASK \
404 (1 << CODING_CATEGORY_SHIFT_JIS)
405 #define CODING_CATEGORY_ISO_7_MASK \
406 (1 << CODING_CATEGORY_ISO_7)
407 #define CODING_CATEGORY_ISO_8_DESIGNATE_MASK \
408 (1 << CODING_CATEGORY_ISO_8_DESIGNATE)
409 #define CODING_CATEGORY_ISO_8_1_MASK \
410 (1 << CODING_CATEGORY_ISO_8_1)
411 #define CODING_CATEGORY_ISO_8_2_MASK \
412 (1 << CODING_CATEGORY_ISO_8_2)
413 #define CODING_CATEGORY_ISO_LOCK_SHIFT_MASK \
414 (1 << CODING_CATEGORY_ISO_LOCK_SHIFT)
415 #define CODING_CATEGORY_BIG5_MASK \
416 (1 << CODING_CATEGORY_BIG5)
417 #define CODING_CATEGORY_NO_CONVERSION_MASK \
418 (1 << CODING_CATEGORY_NO_CONVERSION)
419 #define CODING_CATEGORY_NOT_FINISHED_MASK \
420 (1 << 30)
421
422 extern Lisp_Object make_decoding_input_stream (Lstream *stream,
423 Lisp_Object codesys);
424 extern Lisp_Object make_encoding_input_stream (Lstream *stream,
425 Lisp_Object codesys);
426 extern Lisp_Object make_decoding_output_stream (Lstream *stream,
427 Lisp_Object codesys);
428 extern Lisp_Object make_encoding_output_stream (Lstream *stream,
429 Lisp_Object codesys);
430 extern Lisp_Object decoding_stream_coding_system (Lstream *stream);
431 extern Lisp_Object encoding_stream_coding_system (Lstream *stream);
432 extern void set_decoding_stream_coding_system (Lstream *stream,
433 Lisp_Object codesys);
434 extern void set_encoding_stream_coding_system (Lstream *stream,
435 Lisp_Object codesys);
436
437 /* In mule-ccl.c */
438 int ccl_driver (struct ccl_program *ccl, CONST unsigned char *src,
439 unsigned_char_dynarr *dst, int n, int end_flag);
440 void set_ccl_program (struct ccl_program *ccl, Lisp_Object val, int *regs,
441 int numregs, int initial_ic);
442
443 #endif /* _XEMACS_MULE_CODING_H_ */