|
771
|
1 /* Header for encoding conversion functions; coding-system object.
|
|
|
2 #### rename me to coding-system.h
|
|
428
|
3 Copyright (C) 1991, 1995 Free Software Foundation, Inc.
|
|
|
4 Copyright (C) 1995 Sun Microsystems, Inc.
|
|
793
|
5 Copyright (C) 2000, 2001, 2002 Ben Wing.
|
|
428
|
6
|
|
|
7 This file is part of XEmacs.
|
|
|
8
|
|
|
9 XEmacs is free software; you can redistribute it and/or modify it
|
|
|
10 under the terms of the GNU General Public License as published by the
|
|
|
11 Free Software Foundation; either version 2, or (at your option) any
|
|
|
12 later version.
|
|
|
13
|
|
|
14 XEmacs is distributed in the hope that it will be useful, but WITHOUT
|
|
|
15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
17 for more details.
|
|
|
18
|
|
|
19 You should have received a copy of the GNU General Public License
|
|
|
20 along with XEmacs; see the file COPYING. If not, write to
|
|
|
21 the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
|
22 Boston, MA 02111-1307, USA. */
|
|
|
23
|
|
|
24 /* Synched up with: Mule 2.3. Not in FSF. */
|
|
|
25
|
|
771
|
26 /* Authorship:
|
|
|
27
|
|
|
28 Current primary author: Ben Wing <ben@xemacs.org>
|
|
|
29
|
|
|
30 Written by Ben Wing <ben@xemacs.org> for XEmacs, 1995, loosely based
|
|
|
31 on code written 91.10.09 by K.Handa <handa@etl.go.jp>.
|
|
|
32 Rewritten again 2000-2001 by Ben Wing to support properly
|
|
|
33 abstracted coding systems.
|
|
|
34 September 2001: Finished last part of abstraction, the detection
|
|
|
35 mechanism.
|
|
|
36 */
|
|
428
|
37
|
|
440
|
38 #ifndef INCLUDED_file_coding_h_
|
|
|
39 #define INCLUDED_file_coding_h_
|
|
428
|
40
|
|
771
|
41 /* Capsule description of the different structures, what their purpose is,
|
|
|
42 how they fit together, and where various bits of data are stored.
|
|
|
43
|
|
2297
|
44 A "coding system" is an algorithm for converting stream data in one format
|
|
|
45 into stream data in another format. Currently most of the coding systems
|
|
|
46 we have created concern internationalized text, and convert between the
|
|
|
47 XEmacs internal format for multilingual text, and various external
|
|
771
|
48 representations of such text. However, any such conversion is possible,
|
|
|
49 for example, compressing or uncompressing text using the gzip algorithm.
|
|
|
50 All coding systems provide both encode and decode routines, so that the
|
|
2297
|
51 conversion can go both ways. Unfortunately encoding and decoding may not
|
|
|
52 be exact inverses, even for a specific instance of a coding system. Care
|
|
|
53 must be taken when this is not the case.
|
|
771
|
54
|
|
|
55 The way we handle this is by dividing the various potential coding
|
|
|
56 systems into types, analogous to classes in C++. Each coding system
|
|
|
57 type encompasses a series of related coding systems that it can
|
|
|
58 implement, and it has properties which control how exactly the encoding
|
|
|
59 works. A particular set of values for each of the properties makes up a
|
|
|
60 "coding system", and specifies one particular encoding. A `struct
|
|
|
61 Lisp_Coding_System' object encapsulates those settings -- its type, the
|
|
|
62 values chosen for all properties of that type, a name for the coding
|
|
|
63 system, some documentation.
|
|
|
64
|
|
|
65 In addition, there are of course methods associated with a coding system
|
|
|
66 type, implementing the encoding, decoding, etc. These are stored in a
|
|
|
67 `struct coding_system_methods' object, one per coding-system type, which
|
|
|
68 contains mostly function pointers. This is retrievable from the
|
|
|
69 coding-system object (i.e. the struct Lisp_Coding_System), which has a
|
|
|
70 pointer to it.
|
|
|
71
|
|
|
72 In order to actually use a coding system to do an encoding or decoding
|
|
|
73 operation, you need to use a coding Lstream.
|
|
|
74
|
|
|
75 Now let's look more at attached data. All coding systems have certain
|
|
|
76 common data fields -- name, type, documentation, etc. -- as well as a
|
|
|
77 bunch more that are defined by the coding system type. To handle this
|
|
|
78 cleanly, each coding system type defines a structure that holds just the
|
|
|
79 fields of data particular to it, and calls it e.g. `struct
|
|
|
80 iso2022_coding_system' for coding system type `iso2022'. When the
|
|
|
81 memory block holding the coding system object is created, it is sized
|
|
|
82 such that it can hold both the struct Lisp_Coding_System and the struct
|
|
|
83 iso2022_coding_system (or whatever) directly following it. (This is a
|
|
|
84 common trick; another possibility is to have a void * pointer in the
|
|
|
85 struct Lisp_Coding_System, which points to another memory block holding
|
|
|
86 the struct iso2022_coding_system.) A macro is provided
|
|
|
87 (CODING_SYSTEM_TYPE_DATA) to retrieve a pointer of the right type to the
|
|
|
88 type-specific data contained within the overall `struct
|
|
|
89 Lisp_Coding_System' block.
|
|
|
90
|
|
|
91 Lstreams, similarly, are objects of type `struct lstream' holding data
|
|
|
92 about the stream operation (how much data has been read or written, any
|
|
|
93 buffered data, any error conditions, etc.), and like coding systems have
|
|
|
94 different types. They have a structure called `Lstream_implementation',
|
|
|
95 one per lstream type, exactly analogous to `struct
|
|
|
96 coding_system_methods'. In addition, they have type-specific data
|
|
|
97 (specifying, e.g., the file number, FILE *, memory location, other
|
|
|
98 lstream, etc. to read the data from or write it to, and for conversion
|
|
|
99 processes, the current state of the process -- are we decoding ASCII or
|
|
|
100 Kanji characters? are we in the middle of a processing an escape
|
|
|
101 sequence? etc.). This type-specific data is stored in a structure
|
|
|
102 named `struct coding_stream'. Just like for coding systems, the
|
|
|
103 type-independent data in the `struct lstream' and the type-dependent
|
|
|
104 data in the `struct coding_stream' are stored together in the same
|
|
|
105 memory block.
|
|
428
|
106
|
|
771
|
107 Now things get a bit tricky. The `struct coding_stream' is
|
|
|
108 type-specific from the point of view of an lstream, but not from the
|
|
|
109 point of view of a coding system. It contains only general data about
|
|
|
110 the conversion process, e.g. the name of the coding system used for
|
|
|
111 conversion, the lstream that we take data from or write it to (depending
|
|
|
112 on whether this was created as a read stream or a write stream), a
|
|
|
113 buffer to hold extra data we retrieved but can't send on yet, some
|
|
|
114 flags, etc. It also needs some data specific to the particular coding
|
|
|
115 system and thus to the particular operation going on. This data is held
|
|
|
116 in a structure named (e.g.) `struct iso2022_coding_stream', and it's
|
|
|
117 held in a separate memory block and pointed to by the generic `struct
|
|
|
118 coding_stream'. It's not glommed into a single memory block both
|
|
|
119 because that would require making changes to the generic lstream code
|
|
|
120 and more importantly because the coding system used in a particular
|
|
|
121 coding lstream can be changed at any point during the lifetime of the
|
|
|
122 lstream, and possibly multiple times. (For example, it can be set using
|
|
|
123 the Lisp primitives `set-process-input-coding-system' and
|
|
|
124 `set-console-tty-input-coding-system', as well as getting set when a
|
|
|
125 conversion operation was started with coding system `undecided' and the
|
|
2297
|
126 correct coding system was then detected.) #### This suggests implementing
|
|
|
127 compound text extended segments by saving the state of the ctext stream,
|
|
|
128 and installing an appropriate for the duration of the segment.
|
|
428
|
129
|
|
771
|
130 IMPORTANT NOTE: There are at least two ancillary data structures
|
|
|
131 associated with a coding system type. (There may also be detection data;
|
|
|
132 see elsewhere.) It's important, when writing a coding system type, to
|
|
|
133 keep straight which type of data goes where. In particular, `struct
|
|
|
134 foo_coding_system' is attached to the coding system object itself. This
|
|
|
135 is a permanent object and there's only one per coding system. It's
|
|
|
136 created once, usually at init time, and never destroyed. So, `struct
|
|
|
137 foo_coding_system' should in general not contain dynamic data! (Just
|
|
|
138 data describing the properties of the coding system.) In particular,
|
|
|
139 *NO* data about any conversion in progress. There may be many
|
|
|
140 conversions going on simultaneously using a particular coding system,
|
|
|
141 and by storing conversion data in the coding system, these conversions
|
|
|
142 will overwrite each other's data.
|
|
|
143
|
|
|
144 Instead, use the lstream object, whose purpose is to encapsulate a
|
|
|
145 particular conversion and all associated data. From the lstream object,
|
|
|
146 you can get the struct coding_stream using something like
|
|
|
147
|
|
|
148 struct coding_stream *str = LSTREAM_TYPE_DATA (lstr, coding);
|
|
|
149
|
|
|
150 But usually this structure is already passed to you as one of the
|
|
|
151 parameters of the method being invoked.
|
|
|
152
|
|
|
153 From the struct coding_stream, you can retrieve the
|
|
|
154 coding-system-type-specific data using something like
|
|
|
155
|
|
|
156 struct foo_coding_stream *data = CODING_STREAM_TYPE_DATA (str, foo);
|
|
|
157
|
|
|
158 Then, use this structure to hold all data relevant to the particular
|
|
|
159 conversion being done.
|
|
|
160
|
|
|
161 Initialize this structure whenever init_coding_stream_method is called
|
|
|
162 (this may happen more than once), and finalize it (free resources, etc.)
|
|
|
163 when finalize_coding_stream_method is called.
|
|
|
164 */
|
|
|
165
|
|
|
166 struct coding_stream;
|
|
|
167 struct detection_state;
|
|
|
168
|
|
1204
|
169 extern const struct sized_memory_description coding_system_methods_description;
|
|
771
|
170
|
|
|
171 struct coding_system_methods;
|
|
|
172
|
|
|
173 enum source_sink_type
|
|
428
|
174 {
|
|
771
|
175 DECODES_CHARACTER_TO_BYTE,
|
|
|
176 DECODES_BYTE_TO_BYTE,
|
|
|
177 DECODES_BYTE_TO_CHARACTER,
|
|
|
178 DECODES_CHARACTER_TO_CHARACTER
|
|
428
|
179 };
|
|
|
180
|
|
|
181 enum eol_type
|
|
|
182 {
|
|
|
183 EOL_LF,
|
|
|
184 EOL_CRLF,
|
|
771
|
185 EOL_CR,
|
|
1429
|
186 EOL_AUTODETECT
|
|
428
|
187 };
|
|
|
188
|
|
|
189 struct Lisp_Coding_System
|
|
|
190 {
|
|
|
191 struct lcrecord_header header;
|
|
771
|
192 struct coding_system_methods *methods;
|
|
428
|
193
|
|
1204
|
194 #define CODING_SYSTEM_SLOT_DECLARATION
|
|
|
195 #define MARKED_SLOT(x) Lisp_Object x;
|
|
|
196 #include "coding-system-slots.h"
|
|
771
|
197
|
|
1204
|
198 /* Eol type requested by user. See comment about EOL junk in
|
|
|
199 coding-system-slots.h. */
|
|
771
|
200 enum eol_type eol_type;
|
|
428
|
201
|
|
2132
|
202 /* If true, this is an internal coding system, which will not show up in
|
|
|
203 coding-system-list unless a special parameter is given to it. */
|
|
|
204 int internal_p;
|
|
|
205
|
|
771
|
206 /* type-specific extra data attached to a coding_system */
|
|
|
207 char data[1];
|
|
428
|
208 };
|
|
|
209 typedef struct Lisp_Coding_System Lisp_Coding_System;
|
|
|
210
|
|
440
|
211 DECLARE_LRECORD (coding_system, Lisp_Coding_System);
|
|
|
212 #define XCODING_SYSTEM(x) XRECORD (x, coding_system, Lisp_Coding_System)
|
|
617
|
213 #define wrap_coding_system(p) wrap_record (p, coding_system)
|
|
428
|
214 #define CODING_SYSTEMP(x) RECORDP (x, coding_system)
|
|
|
215 #define CHECK_CODING_SYSTEM(x) CHECK_RECORD (x, coding_system)
|
|
|
216 #define CONCHECK_CODING_SYSTEM(x) CONCHECK_RECORD (x, coding_system)
|
|
|
217
|
|
1204
|
218 enum coding_system_variant
|
|
|
219 {
|
|
|
220 no_conversion_coding_system,
|
|
|
221 convert_eol_coding_system,
|
|
|
222 undecided_coding_system,
|
|
|
223 chain_coding_system,
|
|
|
224 text_file_wrapper_coding_system,
|
|
|
225 internal_coding_system,
|
|
|
226 gzip_coding_system,
|
|
|
227 mswindows_multibyte_to_unicode_coding_system,
|
|
|
228 mswindows_multibyte_coding_system,
|
|
|
229 iso2022_coding_system,
|
|
|
230 ccl_coding_system,
|
|
|
231 shift_jis_coding_system,
|
|
|
232 big5_coding_system,
|
|
1429
|
233 unicode_coding_system
|
|
1204
|
234 };
|
|
|
235
|
|
771
|
236 struct coding_system_methods
|
|
|
237 {
|
|
|
238 Lisp_Object type;
|
|
|
239 Lisp_Object predicate_symbol;
|
|
|
240
|
|
1204
|
241 /* Type expressed as an enum, needed for KKCC marking of the
|
|
|
242 type-specific lstream data; copied into the struct coding_stream. */
|
|
|
243
|
|
|
244 enum coding_system_variant enumtype;
|
|
|
245
|
|
771
|
246 /* Implementation specific methods: */
|
|
|
247
|
|
|
248 /* Init method: Initialize coding-system data. Optional. */
|
|
|
249 void (*init_method) (Lisp_Object coding_system);
|
|
|
250
|
|
|
251 /* Mark method: Mark any Lisp objects in the type-specific data
|
|
|
252 attached to the coding-system object. Optional. */
|
|
|
253 void (*mark_method) (Lisp_Object coding_system);
|
|
|
254
|
|
|
255 /* Print method: Print the type-specific properties of this coding
|
|
|
256 system, as part of `print'-ing the object. If this method is defined
|
|
|
257 and prints anything, it should print a space as the first thing it
|
|
|
258 does. Optional. */
|
|
|
259 void (*print_method) (Lisp_Object cs, Lisp_Object printcharfun,
|
|
|
260 int escapeflag);
|
|
|
261
|
|
|
262 /* Canonicalize method: Convert this coding system to another one; called
|
|
|
263 once, at creation time, after all properties have been parsed. The
|
|
|
264 returned value should be a coding system created with
|
|
|
265 make_internal_coding_system() (passing the existing coding system as the
|
|
|
266 first argument), and will become the coding system returned by
|
|
|
267 `make-coding-system'. Optional.
|
|
|
268
|
|
|
269 NOTE: There are *three* different uses of "canonical" or "canonicalize"
|
|
|
270 w.r.t. coding systems, and it's important to keep them straight.
|
|
|
271
|
|
|
272 1. The canonicalize method. Used to specify a different coding
|
|
|
273 system, used when doing conversions, in place of the actual coding
|
|
|
274 system itself. Stored in the CANONICAL field of a coding system.
|
|
|
275
|
|
|
276 2. The canonicalize-after-coding method. Used to return the encoding
|
|
|
277 that was "actually" used to decode some text, such that this
|
|
|
278 particular encoding can be used to encode the text again with the
|
|
|
279 expectation that the result will be the same as the original encoding.
|
|
|
280 Particularly important with auto-detecting coding systems.
|
|
|
281
|
|
|
282 3. From the perspective of aliases, a "canonical" coding system is one
|
|
|
283 that's not an alias to some other coding system, and "canonicalization"
|
|
|
284 is the process of traversing the alias pointers to find the canonical
|
|
|
285 coding system that's equivalent to the alias.
|
|
|
286 */
|
|
|
287 Lisp_Object (*canonicalize_method) (Lisp_Object coding_system);
|
|
|
288
|
|
|
289 /* Canonicalize after coding method: Convert this coding system to
|
|
|
290 another one, after coding (usually decoding) has finished. This is
|
|
|
291 meant to be used by auto-detecting coding systems, which should return
|
|
|
292 the actually detected coding system. Optional. */
|
|
|
293 Lisp_Object (*canonicalize_after_coding_method)
|
|
|
294 (struct coding_stream *str);
|
|
|
295
|
|
|
296 /* Convert method: Decode or encode the data in SRC of size N, writing
|
|
|
297 the results into the Dynarr DST. If the conversion_end_type method
|
|
|
298 indicates that the source is characters (as opposed to bytes), you are
|
|
|
299 guaranteed to get only whole characters in the data in SRC/N. STR, a
|
|
|
300 struct coding_stream, stores all necessary state and other info about
|
|
|
301 the conversion. Coding-specific state (struct TYPE_coding_stream) can
|
|
|
302 be retrieved from STR using CODING_STREAM_TYPE_DATA(). Return value
|
|
|
303 indicates the number of bytes of the *INPUT* that were converted (not
|
|
|
304 the number of bytes written to the Dynarr!). This can be less than
|
|
|
305 the total amount of input passed in; if so, the remainder is
|
|
|
306 considered "rejected" and will appear again at the beginning of the
|
|
|
307 data passed in the next time the convert method is called. When EOF
|
|
|
308 is returned on the other end and there's no more data, the convert
|
|
|
309 method will be called one last time, STR->eof set and the passed-in
|
|
|
310 data will consist only of any rejected data from the previous
|
|
|
311 call. (At this point, file handles and similar resources can be
|
|
|
312 closed, but do NOT arbitrarily free data structures in the
|
|
|
313 type-specific data, because there are operations that can be done on
|
|
|
314 closed streams to query the results of the processing -- specifically,
|
|
|
315 for coding streams, there's the canonicalize_after_coding() method.)
|
|
|
316 Required. */
|
|
|
317 Bytecount (*convert_method) (struct coding_stream *str,
|
|
|
318 const unsigned char *src,
|
|
|
319 unsigned_char_dynarr *dst, Bytecount n);
|
|
|
320
|
|
|
321 /* Coding mark method: Mark any Lisp objects in the type-specific data
|
|
|
322 attached to `struct coding_stream'. Optional. */
|
|
|
323 void (*mark_coding_stream_method) (struct coding_stream *str);
|
|
|
324
|
|
|
325 /* Init coding stream method: Initialize the type-specific data attached
|
|
|
326 to the coding stream (i.e. in struct TYPE_coding_stream), when the
|
|
|
327 coding stream is opened. The type-specific data will be zeroed out.
|
|
|
328 Optional. */
|
|
|
329 void (*init_coding_stream_method) (struct coding_stream *str);
|
|
|
330
|
|
|
331 /* Rewind coding stream method: Reset any necessary type-specific data as
|
|
|
332 a result of the stream being rewound. Optional. */
|
|
|
333 void (*rewind_coding_stream_method) (struct coding_stream *str);
|
|
|
334
|
|
|
335 /* Finalize coding stream method: Clean up the type-specific data
|
|
|
336 attached to the coding stream (i.e. in struct TYPE_coding_stream).
|
|
|
337 Happens when the Lstream is deleted using Lstream_delete() or is
|
|
|
338 garbage-collected. Most streams are deleted after they've been used,
|
|
|
339 so it's less likely (but still possible) that allocated data will
|
|
|
340 stick around until GC time. (File handles can also be closed when EOF
|
|
|
341 is signalled; but some data must stick around after this point, for
|
|
|
342 the benefit of canonicalize_after_coding. See the convert method.)
|
|
|
343 Called only once (NOT called at disksave time). Optional. */
|
|
|
344 void (*finalize_coding_stream_method) (struct coding_stream *str);
|
|
|
345
|
|
|
346 /* Finalize method: Clean up type-specific data (e.g. free allocated
|
|
|
347 data) attached to the coding system (i.e. in struct
|
|
|
348 TYPE_coding_system), when the coding system is about to be garbage
|
|
|
349 collected. (Currently not called.) Called only once (NOT called at
|
|
|
350 disksave time). Optional. */
|
|
|
351 void (*finalize_method) (Lisp_Object codesys);
|
|
|
352
|
|
|
353 /* Conversion end type method: Does this coding system encode bytes ->
|
|
|
354 characters, characters -> characters, bytes -> bytes, or
|
|
|
355 characters -> bytes?. Default is characters -> bytes. Optional. */
|
|
|
356 enum source_sink_type (*conversion_end_type_method) (Lisp_Object codesys);
|
|
|
357
|
|
|
358 /* Putprop method: Set the value of a type-specific property. If
|
|
|
359 the property name is unrecognized, return 0. If the value is disallowed
|
|
|
360 or erroneous, signal an error. Currently called only at creation time.
|
|
|
361 Optional. */
|
|
|
362 int (*putprop_method) (Lisp_Object codesys,
|
|
|
363 Lisp_Object key,
|
|
|
364 Lisp_Object value);
|
|
|
365
|
|
|
366 /* Getprop method: Return the value of a type-specific property. If
|
|
|
367 the property name is unrecognized, return Qunbound. Optional.
|
|
|
368 */
|
|
|
369 Lisp_Object (*getprop_method) (Lisp_Object coding_system,
|
|
|
370 Lisp_Object prop);
|
|
|
371
|
|
|
372 /* These next three are set as part of the call to
|
|
|
373 INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA. */
|
|
|
374
|
|
|
375 /* Description of the extra data (struct foo_coding_system) attached to a
|
|
1204
|
376 coding system, for pdump purposes. */
|
|
|
377 const struct sized_memory_description *extra_description;
|
|
771
|
378 /* size of struct foo_coding_system -- extra data associated with
|
|
|
379 the coding system */
|
|
|
380 int extra_data_size;
|
|
|
381 /* size of struct foo_coding_stream -- extra data associated with the
|
|
|
382 struct coding_stream, needed for each active coding process
|
|
|
383 using this coding system. note that we can have more than one
|
|
|
384 process active at once (simply by creating more than one coding
|
|
|
385 lstream using this coding system), so we can't store this data in
|
|
|
386 the coding system object. */
|
|
|
387 int coding_data_size;
|
|
|
388 };
|
|
|
389
|
|
|
390 /***** Calling a coding-system method *****/
|
|
|
391
|
|
|
392 #define RAW_CODESYSMETH(cs, m) ((cs)->methods->m##_method)
|
|
|
393 #define HAS_CODESYSMETH_P(cs, m) (!!RAW_CODESYSMETH (cs, m))
|
|
|
394 #define CODESYSMETH(cs, m, args) (((cs)->methods->m##_method) args)
|
|
|
395
|
|
|
396 /* Call a void-returning coding-system method, if it exists. */
|
|
|
397 #define MAYBE_CODESYSMETH(cs, m, args) do { \
|
|
|
398 Lisp_Coding_System *maybe_codesysmeth_cs = (cs); \
|
|
|
399 if (HAS_CODESYSMETH_P (maybe_codesysmeth_cs, m)) \
|
|
|
400 CODESYSMETH (maybe_codesysmeth_cs, m, args); \
|
|
|
401 } while (0)
|
|
|
402
|
|
|
403 /* Call a coding-system method, if it exists, or return GIVEN.
|
|
|
404 NOTE: Multiply-evaluates CS. */
|
|
|
405 #define CODESYSMETH_OR_GIVEN(cs, m, args, given) \
|
|
|
406 (HAS_CODESYSMETH_P (cs, m) ? \
|
|
|
407 CODESYSMETH (cs, m, args) : (given))
|
|
|
408
|
|
|
409 #define XCODESYSMETH(cs, m, args) \
|
|
|
410 CODESYSMETH (XCODING_SYSTEM (cs), m, args)
|
|
|
411 #define MAYBE_XCODESYSMETH(cs, m, args) \
|
|
|
412 MAYBE_CODESYSMETH (XCODING_SYSTEM (cs), m, args)
|
|
|
413 #define XCODESYSMETH_OR_GIVEN(cs, m, args, given) \
|
|
|
414 CODESYSMETH_OR_GIVEN (XCODING_SYSTEM (cs), m, args, given)
|
|
|
415
|
|
|
416
|
|
|
417 /***** Defining new coding-system types *****/
|
|
|
418
|
|
1204
|
419 extern const struct sized_memory_description coding_system_empty_extra_description;
|
|
771
|
420
|
|
800
|
421 #ifdef ERROR_CHECK_TYPES
|
|
771
|
422 #define DECLARE_CODING_SYSTEM_TYPE(type) \
|
|
|
423 \
|
|
|
424 extern struct coding_system_methods * type##_coding_system_methods; \
|
|
826
|
425 DECLARE_INLINE_HEADER ( \
|
|
|
426 struct type##_coding_system * \
|
|
771
|
427 error_check_##type##_coding_system_data (Lisp_Coding_System *cs) \
|
|
826
|
428 ) \
|
|
771
|
429 { \
|
|
|
430 assert (CODING_SYSTEM_TYPE_P (cs, type)); \
|
|
|
431 /* Catch accidental use of INITIALIZE_CODING_SYSTEM_TYPE in place \
|
|
|
432 of INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA. */ \
|
|
|
433 assert (cs->methods->extra_data_size > 0); \
|
|
|
434 return (struct type##_coding_system *) cs->data; \
|
|
|
435 } \
|
|
|
436 \
|
|
826
|
437 DECLARE_INLINE_HEADER ( \
|
|
|
438 struct type##_coding_stream * \
|
|
771
|
439 error_check_##type##_coding_stream_data (struct coding_stream *s) \
|
|
826
|
440 ) \
|
|
771
|
441 { \
|
|
|
442 assert (XCODING_SYSTEM_TYPE_P (s->codesys, type)); \
|
|
|
443 return (struct type##_coding_stream *) s->data; \
|
|
|
444 } \
|
|
|
445 \
|
|
826
|
446 DECLARE_INLINE_HEADER ( \
|
|
|
447 Lisp_Coding_System * \
|
|
771
|
448 error_check_##type##_coding_system_type (Lisp_Object obj) \
|
|
826
|
449 ) \
|
|
771
|
450 { \
|
|
|
451 Lisp_Coding_System *cs = XCODING_SYSTEM (obj); \
|
|
|
452 assert (CODING_SYSTEM_TYPE_P (cs, type)); \
|
|
|
453 return cs; \
|
|
|
454 } \
|
|
|
455 \
|
|
|
456 DECLARE_NOTHING
|
|
|
457 #else
|
|
|
458 #define DECLARE_CODING_SYSTEM_TYPE(type) \
|
|
|
459 extern struct coding_system_methods * type##_coding_system_methods
|
|
800
|
460 #endif /* ERROR_CHECK_TYPES */
|
|
771
|
461
|
|
|
462 #define DEFINE_CODING_SYSTEM_TYPE(type) \
|
|
|
463 struct coding_system_methods * type##_coding_system_methods
|
|
|
464
|
|
1204
|
465 #define DEFINE_CODING_SYSTEM_TYPE_WITH_DATA(type) \
|
|
|
466 struct coding_system_methods * type##_coding_system_methods; \
|
|
|
467 static const struct sized_memory_description \
|
|
|
468 type##_coding_system_description_0 = { \
|
|
|
469 sizeof (struct type##_coding_system), \
|
|
|
470 type##_coding_system_description \
|
|
|
471 }
|
|
|
472
|
|
771
|
473 #define INITIALIZE_CODING_SYSTEM_TYPE(ty, pred_sym) do { \
|
|
|
474 ty##_coding_system_methods = \
|
|
|
475 xnew_and_zero (struct coding_system_methods); \
|
|
|
476 ty##_coding_system_methods->type = Q##ty; \
|
|
|
477 ty##_coding_system_methods->extra_description = \
|
|
1204
|
478 &coding_system_empty_extra_description; \
|
|
|
479 ty##_coding_system_methods->enumtype = ty##_coding_system; \
|
|
771
|
480 defsymbol_nodump (&ty##_coding_system_methods->predicate_symbol, \
|
|
|
481 pred_sym); \
|
|
|
482 add_entry_to_coding_system_type_list (ty##_coding_system_methods); \
|
|
2367
|
483 dump_add_root_block_ptr (&ty##_coding_system_methods, \
|
|
771
|
484 &coding_system_methods_description); \
|
|
|
485 } while (0)
|
|
|
486
|
|
|
487 #define REINITIALIZE_CODING_SYSTEM_TYPE(type) do { \
|
|
|
488 staticpro_nodump (&type##_coding_system_methods->predicate_symbol); \
|
|
|
489 } while (0)
|
|
|
490
|
|
|
491 /* This assumes the existence of two structures:
|
|
|
492
|
|
|
493 struct foo_coding_system (attached to the coding system)
|
|
|
494 struct foo_coding_stream (per coding process, attached to the
|
|
|
495 struct coding_stream)
|
|
1204
|
496 const struct memory_description foo_coding_system_description[]
|
|
|
497 (data description of struct foo_coding_system)
|
|
771
|
498
|
|
1204
|
499 For an example of how to do the description, see
|
|
771
|
500 chain_coding_system_description.
|
|
|
501 */
|
|
|
502 #define INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA(type, pred_sym) \
|
|
|
503 do { \
|
|
|
504 INITIALIZE_CODING_SYSTEM_TYPE (type, pred_sym); \
|
|
|
505 type##_coding_system_methods->extra_data_size = \
|
|
|
506 sizeof (struct type##_coding_system); \
|
|
|
507 type##_coding_system_methods->extra_description = \
|
|
1204
|
508 &type##_coding_system_description_0; \
|
|
771
|
509 type##_coding_system_methods->coding_data_size = \
|
|
|
510 sizeof (struct type##_coding_stream); \
|
|
|
511 } while (0)
|
|
|
512
|
|
|
513 /* Declare that coding-system-type TYPE has method METH; used in
|
|
|
514 initialization routines */
|
|
|
515 #define CODING_SYSTEM_HAS_METHOD(type, meth) \
|
|
|
516 (type##_coding_system_methods->meth##_method = type##_##meth)
|
|
|
517
|
|
|
518 /***** Macros for accessing coding-system types *****/
|
|
|
519
|
|
|
520 #define CODING_SYSTEM_TYPE_P(cs, type) \
|
|
|
521 ((cs)->methods == type##_coding_system_methods)
|
|
|
522 #define XCODING_SYSTEM_TYPE_P(cs, type) \
|
|
|
523 CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (cs), type)
|
|
|
524
|
|
800
|
525 #ifdef ERROR_CHECK_TYPES
|
|
771
|
526 # define CODING_SYSTEM_TYPE_DATA(cs, type) \
|
|
|
527 error_check_##type##_coding_system_data (cs)
|
|
|
528 #else
|
|
|
529 # define CODING_SYSTEM_TYPE_DATA(cs, type) \
|
|
|
530 ((struct type##_coding_system *) \
|
|
|
531 (cs)->data)
|
|
|
532 #endif
|
|
|
533
|
|
|
534 #define XCODING_SYSTEM_TYPE_DATA(cs, type) \
|
|
|
535 CODING_SYSTEM_TYPE_DATA (XCODING_SYSTEM_OF_TYPE (cs, type), type)
|
|
|
536
|
|
800
|
537 #ifdef ERROR_CHECK_TYPES
|
|
771
|
538 # define XCODING_SYSTEM_OF_TYPE(x, type) \
|
|
|
539 error_check_##type##_coding_system_type (x)
|
|
|
540 # define XSETCODING_SYSTEM_OF_TYPE(x, p, type) do \
|
|
|
541 { \
|
|
793
|
542 x = wrap_coding_system (p); \
|
|
|
543 assert (CODING_SYSTEM_TYPEP (XCODING_SYSTEM (x), type)); \
|
|
771
|
544 } while (0)
|
|
|
545 #else
|
|
|
546 # define XCODING_SYSTEM_OF_TYPE(x, type) XCODING_SYSTEM (x)
|
|
793
|
547 # define XSETCODING_SYSTEM_OF_TYPE(x, p, type) do \
|
|
|
548 { \
|
|
|
549 x = wrap_coding_system (p); \
|
|
|
550 } while (0)
|
|
771
|
551 #endif /* ERROR_CHECK_TYPE_CHECK */
|
|
|
552
|
|
|
553 #define CODING_SYSTEM_TYPEP(x, type) \
|
|
|
554 (CODING_SYSTEMP (x) && CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (x), type))
|
|
|
555 #define CHECK_CODING_SYSTEM_OF_TYPE(x, type) do { \
|
|
|
556 CHECK_CODING_SYSTEM (x); \
|
|
|
557 if (!CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (x), type)) \
|
|
|
558 dead_wrong_type_argument \
|
|
|
559 (type##_coding_system_methods->predicate_symbol, x); \
|
|
|
560 } while (0)
|
|
|
561 #define CONCHECK_CODING_SYSTEM_OF_TYPE(x, type) do { \
|
|
|
562 CONCHECK_CODING_SYSTEM (x); \
|
|
|
563 if (!(CODING_SYSTEM_TYPEP (x, type))) \
|
|
|
564 x = wrong_type_argument \
|
|
|
565 (type##_coding_system_methods->predicate_symbol, x); \
|
|
|
566 } while (0)
|
|
|
567
|
|
|
568 #define CODING_SYSTEM_METHODS(codesys) ((codesys)->methods)
|
|
428
|
569 #define CODING_SYSTEM_NAME(codesys) ((codesys)->name)
|
|
771
|
570 #define CODING_SYSTEM_DESCRIPTION(codesys) ((codesys)->description)
|
|
|
571 #define CODING_SYSTEM_TYPE(codesys) ((codesys)->methods->type)
|
|
428
|
572 #define CODING_SYSTEM_MNEMONIC(codesys) ((codesys)->mnemonic)
|
|
771
|
573 #define CODING_SYSTEM_DOCUMENTATION(codesys) ((codesys)->documentation)
|
|
428
|
574 #define CODING_SYSTEM_POST_READ_CONVERSION(codesys) \
|
|
|
575 ((codesys)->post_read_conversion)
|
|
|
576 #define CODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
|
|
|
577 ((codesys)->pre_write_conversion)
|
|
|
578 #define CODING_SYSTEM_EOL_TYPE(codesys) ((codesys)->eol_type)
|
|
771
|
579 #define CODING_SYSTEM_EOL_LF(codesys) ((codesys)->eol[EOL_LF])
|
|
|
580 #define CODING_SYSTEM_EOL_CRLF(codesys) ((codesys)->eol[EOL_CRLF])
|
|
|
581 #define CODING_SYSTEM_EOL_CR(codesys) ((codesys)->eol[EOL_CR])
|
|
|
582 #define CODING_SYSTEM_TEXT_FILE_WRAPPER(codesys) ((codesys)->text_file_wrapper)
|
|
|
583 #define CODING_SYSTEM_AUTO_EOL_WRAPPER(codesys) ((codesys)->auto_eol_wrapper)
|
|
|
584 #define CODING_SYSTEM_SUBSIDIARY_PARENT(codesys) ((codesys)->subsidiary_parent)
|
|
|
585 #define CODING_SYSTEM_CANONICAL(codesys) ((codesys)->canonical)
|
|
428
|
586
|
|
771
|
587 #define CODING_SYSTEM_CHAIN_CHAIN(codesys) \
|
|
|
588 (CODING_SYSTEM_TYPE_DATA (codesys, chain)->chain)
|
|
|
589 #define CODING_SYSTEM_CHAIN_COUNT(codesys) \
|
|
|
590 (CODING_SYSTEM_TYPE_DATA (codesys, chain)->count)
|
|
|
591 #define CODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING(codesys) \
|
|
|
592 (CODING_SYSTEM_TYPE_DATA (codesys, chain)->canonicalize_after_coding)
|
|
428
|
593
|
|
771
|
594 #define XCODING_SYSTEM_METHODS(codesys) \
|
|
|
595 CODING_SYSTEM_METHODS (XCODING_SYSTEM (codesys))
|
|
428
|
596 #define XCODING_SYSTEM_NAME(codesys) \
|
|
|
597 CODING_SYSTEM_NAME (XCODING_SYSTEM (codesys))
|
|
771
|
598 #define XCODING_SYSTEM_DESCRIPTION(codesys) \
|
|
|
599 CODING_SYSTEM_DESCRIPTION (XCODING_SYSTEM (codesys))
|
|
428
|
600 #define XCODING_SYSTEM_TYPE(codesys) \
|
|
|
601 CODING_SYSTEM_TYPE (XCODING_SYSTEM (codesys))
|
|
|
602 #define XCODING_SYSTEM_MNEMONIC(codesys) \
|
|
|
603 CODING_SYSTEM_MNEMONIC (XCODING_SYSTEM (codesys))
|
|
771
|
604 #define XCODING_SYSTEM_DOCUMENTATION(codesys) \
|
|
|
605 CODING_SYSTEM_DOCUMENTATION (XCODING_SYSTEM (codesys))
|
|
428
|
606 #define XCODING_SYSTEM_POST_READ_CONVERSION(codesys) \
|
|
|
607 CODING_SYSTEM_POST_READ_CONVERSION (XCODING_SYSTEM (codesys))
|
|
|
608 #define XCODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
|
|
|
609 CODING_SYSTEM_PRE_WRITE_CONVERSION (XCODING_SYSTEM (codesys))
|
|
|
610 #define XCODING_SYSTEM_EOL_TYPE(codesys) \
|
|
|
611 CODING_SYSTEM_EOL_TYPE (XCODING_SYSTEM (codesys))
|
|
|
612 #define XCODING_SYSTEM_EOL_LF(codesys) \
|
|
|
613 CODING_SYSTEM_EOL_LF (XCODING_SYSTEM (codesys))
|
|
|
614 #define XCODING_SYSTEM_EOL_CRLF(codesys) \
|
|
|
615 CODING_SYSTEM_EOL_CRLF (XCODING_SYSTEM (codesys))
|
|
|
616 #define XCODING_SYSTEM_EOL_CR(codesys) \
|
|
|
617 CODING_SYSTEM_EOL_CR (XCODING_SYSTEM (codesys))
|
|
771
|
618 #define XCODING_SYSTEM_TEXT_FILE_WRAPPER(codesys) \
|
|
|
619 CODING_SYSTEM_TEXT_FILE_WRAPPER (XCODING_SYSTEM (codesys))
|
|
|
620 #define XCODING_SYSTEM_AUTO_EOL_WRAPPER(codesys) \
|
|
|
621 CODING_SYSTEM_AUTO_EOL_WRAPPER (XCODING_SYSTEM (codesys))
|
|
|
622 #define XCODING_SYSTEM_SUBSIDIARY_PARENT(codesys) \
|
|
|
623 CODING_SYSTEM_SUBSIDIARY_PARENT (XCODING_SYSTEM (codesys))
|
|
|
624 #define XCODING_SYSTEM_CANONICAL(codesys) \
|
|
|
625 CODING_SYSTEM_CANONICAL (XCODING_SYSTEM (codesys))
|
|
428
|
626
|
|
771
|
627 #define XCODING_SYSTEM_CHAIN_CHAIN(codesys) \
|
|
|
628 CODING_SYSTEM_CHAIN_CHAIN (XCODING_SYSTEM (codesys))
|
|
|
629 #define XCODING_SYSTEM_CHAIN_COUNT(codesys) \
|
|
|
630 CODING_SYSTEM_CHAIN_COUNT (XCODING_SYSTEM (codesys))
|
|
|
631 #define XCODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING(codesys) \
|
|
|
632 CODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING (XCODING_SYSTEM (codesys))
|
|
428
|
633
|
|
771
|
634 /**************************************************/
|
|
|
635 /* Detection */
|
|
|
636 /**************************************************/
|
|
428
|
637
|
|
771
|
638 #define MAX_DETECTOR_CATEGORIES 256
|
|
|
639 #define MAX_DETECTORS 64
|
|
428
|
640
|
|
771
|
641 #define MAX_BYTES_PROCESSED_FOR_DETECTION 65536
|
|
428
|
642
|
|
771
|
643 struct detection_state
|
|
428
|
644 {
|
|
771
|
645 int seen_non_ascii;
|
|
|
646 Bytecount bytes_seen;
|
|
428
|
647
|
|
771
|
648 char categories[MAX_DETECTOR_CATEGORIES];
|
|
|
649 Bytecount data_offset[MAX_DETECTORS];
|
|
|
650 /* ... more data follows; data_offset[detector_##TYPE] points to
|
|
|
651 the data for that type */
|
|
428
|
652 };
|
|
|
653
|
|
771
|
654 #define DETECTION_STATE_DATA(st, type) \
|
|
|
655 ((struct type##_detector *) \
|
|
|
656 ((char *) (st) + (st)->data_offset[detector_##type]))
|
|
428
|
657
|
|
448
|
658 /* Distinguishable categories of encodings.
|
|
|
659
|
|
|
660 This list determines the initial priority of the categories.
|
|
|
661
|
|
|
662 For better or worse, currently Mule files are encoded in 7-bit ISO 2022.
|
|
|
663 For this reason, under Mule ISO_7 gets highest priority.
|
|
|
664
|
|
|
665 Putting NO_CONVERSION second prevents "binary corruption" in the
|
|
|
666 default case in all but the (presumably) extremely rare case of a
|
|
|
667 binary file which contains redundant escape sequences but no 8-bit
|
|
|
668 characters.
|
|
|
669
|
|
|
670 The remaining priorities are based on perceived "internationalization
|
|
|
671 political correctness." An exception is UCS-4 at the bottom, since
|
|
|
672 basically everything is compatible with UCS-4, but it is likely to
|
|
|
673 be very rare as an external encoding. */
|
|
|
674
|
|
771
|
675 /* Macros to define code of control characters for ISO2022's functions. */
|
|
|
676 /* Used by the detection routines of other coding system types as well. */
|
|
|
677 /* code */ /* function */
|
|
|
678 #define ISO_CODE_LF 0x0A /* line-feed */
|
|
|
679 #define ISO_CODE_CR 0x0D /* carriage-return */
|
|
|
680 #define ISO_CODE_SO 0x0E /* shift-out */
|
|
|
681 #define ISO_CODE_SI 0x0F /* shift-in */
|
|
|
682 #define ISO_CODE_ESC 0x1B /* escape */
|
|
|
683 #define ISO_CODE_DEL 0x7F /* delete */
|
|
|
684 #define ISO_CODE_SS2 0x8E /* single-shift-2 */
|
|
|
685 #define ISO_CODE_SS3 0x8F /* single-shift-3 */
|
|
|
686 #define ISO_CODE_CSI 0x9B /* control-sequence-introduce */
|
|
|
687
|
|
|
688 enum detection_result
|
|
|
689 {
|
|
|
690 /* Basically means a magic cookie was seen indicating this type, or
|
|
|
691 something similar. */
|
|
|
692 DET_NEAR_CERTAINTY = 4,
|
|
|
693 DET_HIGHEST = 4,
|
|
|
694 /* Characteristics seen that are unlikely to be other coding system types
|
|
|
695 -- e.g. ISO-2022 escape sequences, or perhaps a consistent pattern of
|
|
|
696 alternating zero bytes in UTF-16, along with Unicode LF or CRLF
|
|
|
697 sequences at regular intervals. (Zero bytes are unlikely or impossible
|
|
|
698 in most text encodings.) */
|
|
|
699 DET_QUITE_PROBABLE = 3,
|
|
|
700 /* Strong or medium statistical likelihood. At least some
|
|
|
701 characteristics seen that match what's normally found in this encoding
|
|
|
702 -- e.g. in Shift-JIS, a number of two-byte Japanese character
|
|
|
703 sequences in the right range, and nothing out of range; or in Unicode,
|
|
|
704 much higher statistical variance in the odd bytes than in the even
|
|
|
705 bytes, or vice-versa (perhaps the presence of regular EOL sequences
|
|
|
706 would bump this too to DET_QUITE_PROBABLE). This is quite often a
|
|
|
707 statistical test. */
|
|
|
708 DET_SOMEWHAT_LIKELY = 2,
|
|
|
709 /* Weak statistical likelihood. Pretty much any features at all that
|
|
|
710 characterize this encoding, and nothing that rules against it. */
|
|
|
711 DET_SLIGHTLY_LIKELY = 1,
|
|
|
712 /* Default state. Perhaps it indicates pure ASCII or something similarly
|
|
|
713 vague seen in Shift-JIS, or, exactly as the level says, it might mean
|
|
|
714 in a statistical-based detector that the pros and cons are balanced
|
|
|
715 out. This is also the lowest level that will be accepted by the
|
|
|
716 auto-detector without asking the user: If all available detectors
|
|
|
717 report lower levels for all categories with attached coding systems,
|
|
|
718 the user will be shown the results and explicitly prompted for action.
|
|
|
719 The user will also be prompted if this is the highest available level
|
|
|
720 and more than one detector reports the level. (See below about the
|
|
|
721 consequent necessity of an "ASCII" detector, which will return level 1
|
|
|
722 or higher for most plain text files.) */
|
|
|
723 DET_AS_LIKELY_AS_UNLIKELY = 0,
|
|
|
724 /* Some characteristics seen that are unusual for this encoding --
|
|
|
725 e.g. unusual control characters in a plain-text encoding, lots of
|
|
|
726 8-bit characters, or little statistical variance in the odd and even
|
|
|
727 bytes in UTF-16. */
|
|
|
728 DET_SOMEWHAT_UNLIKELY = -1,
|
|
|
729 /* This indicates that there is very little chance the data is in the
|
|
|
730 right format; this is probably the lowest level you can get when
|
|
|
731 presenting random binary data to a text file, because there are no
|
|
|
732 "specific sequences" you can see that would totally rule out
|
|
|
733 recognition. */
|
|
|
734 DET_QUITE_IMPROBABLE = -2,
|
|
|
735 /* An erroneous sequence was seen. */
|
|
|
736 DET_NEARLY_IMPOSSIBLE = -3,
|
|
1429
|
737 DET_LOWEST = -3
|
|
771
|
738 };
|
|
|
739
|
|
|
740 extern int coding_detector_count;
|
|
|
741 extern int coding_detector_category_count;
|
|
|
742
|
|
|
743 struct detector_category
|
|
428
|
744 {
|
|
771
|
745 int id;
|
|
|
746 Lisp_Object sym;
|
|
|
747 };
|
|
|
748
|
|
|
749 typedef struct
|
|
|
750 {
|
|
|
751 Dynarr_declare (struct detector_category);
|
|
|
752 } detector_category_dynarr;
|
|
|
753
|
|
|
754 struct detector
|
|
|
755 {
|
|
|
756 int id;
|
|
|
757 detector_category_dynarr *cats;
|
|
|
758 Bytecount data_size;
|
|
|
759 /* Detect method: Required. */
|
|
|
760 void (*detect_method) (struct detection_state *st,
|
|
|
761 const unsigned char *src, Bytecount n);
|
|
|
762 /* Finalize detection state method: Clean up any allocated data in the
|
|
|
763 detection state. Called only once (NOT called at disksave time).
|
|
|
764 Optional. */
|
|
|
765 void (*finalize_detection_state_method) (struct detection_state *st);
|
|
428
|
766 };
|
|
|
767
|
|
771
|
768 /* Lvalue for a particular detection result -- detection state ST,
|
|
|
769 category CAT */
|
|
|
770 #define DET_RESULT(st, cat) ((st)->categories[detector_category_##cat])
|
|
|
771 /* In state ST, set all detection results associated with detector DET to
|
|
|
772 RESULT. */
|
|
|
773 #define SET_DET_RESULTS(st, det, result) \
|
|
|
774 set_detection_results (st, detector_##det, result)
|
|
|
775
|
|
|
776 typedef struct
|
|
|
777 {
|
|
|
778 Dynarr_declare (struct detector);
|
|
|
779 } detector_dynarr;
|
|
|
780
|
|
|
781 extern detector_dynarr *all_coding_detectors;
|
|
|
782
|
|
|
783 #define DEFINE_DETECTOR_CATEGORY(detector, cat) \
|
|
|
784 int detector_category_##cat
|
|
|
785 #define DECLARE_DETECTOR_CATEGORY(detector, cat) \
|
|
|
786 extern int detector_category_##cat
|
|
|
787 #define INITIALIZE_DETECTOR_CATEGORY(detector, cat) \
|
|
|
788 do { \
|
|
|
789 struct detector_category dog; \
|
|
|
790 xzero (dog); \
|
|
|
791 detector_category_##cat = coding_detector_category_count++; \
|
|
|
792 dump_add_opaque_int (&detector_category_##cat); \
|
|
|
793 dog.id = detector_category_##cat; \
|
|
|
794 dog.sym = Q##cat; \
|
|
|
795 Dynarr_add (Dynarr_at (all_coding_detectors, detector_##detector).cats, \
|
|
|
796 dog); \
|
|
|
797 } while (0)
|
|
|
798
|
|
|
799 #define DEFINE_DETECTOR(Detector) \
|
|
|
800 int detector_##Detector
|
|
|
801 #define DECLARE_DETECTOR(Detector) \
|
|
|
802 extern int detector_##Detector
|
|
|
803 #define INITIALIZE_DETECTOR(Detector) \
|
|
|
804 do { \
|
|
|
805 struct detector det; \
|
|
|
806 xzero (det); \
|
|
|
807 detector_##Detector = coding_detector_count++; \
|
|
|
808 dump_add_opaque_int (&detector_##Detector); \
|
|
|
809 det.id = detector_##Detector; \
|
|
|
810 det.cats = Dynarr_new2 (detector_category_dynarr, \
|
|
|
811 struct detector_category); \
|
|
|
812 det.data_size = sizeof (struct Detector##_detector); \
|
|
|
813 Dynarr_add (all_coding_detectors, det); \
|
|
|
814 } while (0)
|
|
|
815 #define DETECTOR_HAS_METHOD(Detector, Meth) \
|
|
|
816 Dynarr_at (all_coding_detectors, detector_##Detector).Meth##_method = \
|
|
802
|
817 Detector##_##Meth
|
|
771
|
818
|
|
|
819
|
|
|
820 /**************************************************/
|
|
|
821 /* Decoding/Encoding */
|
|
|
822 /**************************************************/
|
|
|
823
|
|
|
824 /* Is the source (SOURCEP == 1) or sink (SOURCEP == 0) when encoding specified
|
|
|
825 in characters? */
|
|
|
826
|
|
|
827 enum source_or_sink
|
|
|
828 {
|
|
|
829 CODING_SOURCE,
|
|
|
830 CODING_SINK
|
|
|
831 };
|
|
|
832
|
|
|
833 enum encode_decode
|
|
|
834 {
|
|
|
835 CODING_ENCODE,
|
|
|
836 CODING_DECODE
|
|
|
837 };
|
|
|
838
|
|
|
839 /* Data structure attached to an lstream of type `coding',
|
|
|
840 containing values specific to the coding process. Additional
|
|
|
841 data is stored in the DATA field below; the exact form of that data
|
|
|
842 is controlled by the type of the coding system that governs the
|
|
|
843 conversion (field CODESYS). CODESYS may be set at any time
|
|
|
844 throughout the lifetime of the lstream and possibly more than once.
|
|
|
845 See long comment above for more info. */
|
|
|
846
|
|
|
847 struct coding_stream
|
|
|
848 {
|
|
1204
|
849 /* Enumerated constant listing which type of console this is (TTY, X,
|
|
|
850 MS-Windows, etc.). This duplicates the method structure in
|
|
|
851 XCODING_SYSTEM (str->codesys)->methods->type, which formerly was the
|
|
|
852 only way to determine the coding system type. We need this constant
|
|
|
853 now for KKCC, so that it can be used in an XD_UNION clause to
|
|
|
854 determine the Lisp objects in the type-specific data. */
|
|
|
855 enum coding_system_variant type;
|
|
|
856
|
|
771
|
857 /* Coding system that governs the conversion. */
|
|
|
858 Lisp_Object codesys;
|
|
|
859 /* Original coding system, pre-canonicalization. */
|
|
|
860 Lisp_Object orig_codesys;
|
|
|
861
|
|
|
862 /* Back pointer to current stream. */
|
|
|
863 Lstream *us;
|
|
|
864
|
|
|
865 /* Stream that we read the unprocessed data from or write the processed
|
|
|
866 data to. */
|
|
|
867 Lstream *other_end;
|
|
|
868
|
|
|
869 /* In order to handle both reading to and writing from a coding stream,
|
|
|
870 we phrase the conversion methods like write methods -- we can
|
|
|
871 implement reading in terms of a write method but not vice-versa,
|
|
|
872 because the write method is forced to take only what it's given but
|
|
|
873 the read method can read more data from the other end if necessary.
|
|
|
874 On the other hand, the write method is free to generate all the data
|
|
2297
|
875 it wants (and just write it to the other end), but the read method
|
|
771
|
876 can return only as much as was asked for, so we need to implement our
|
|
|
877 own buffering. */
|
|
|
878
|
|
|
879 /* If we are reading, then we can return only a fixed amount of data, but
|
|
|
880 the converter is free to return as much as it wants, so we direct it
|
|
|
881 to store the data here and lop off chunks as we need them. If we are
|
|
|
882 writing, we use this because the converter takes a Dynarr but we are
|
|
|
883 supposed to write into a fixed buffer. (NOTE: This introduces an extra
|
|
|
884 memory copy.) */
|
|
|
885 unsigned_char_dynarr *convert_to;
|
|
|
886
|
|
|
887 /* The conversion method might reject some of the data -- this typically
|
|
|
888 includes partial characters, partial escape sequences, etc. When
|
|
|
889 writing, we just pass the rejection up to the Lstream module, and it
|
|
|
890 will buffer the data. When reading, however, we need to do the
|
|
|
891 buffering ourselves, and we put it here, combined with newly read
|
|
|
892 data. */
|
|
|
893 unsigned_char_dynarr *convert_from;
|
|
|
894
|
|
|
895 /* If set, this is the last chunk of data being processed. When this is
|
|
|
896 finished, output any necessary terminating control characters, escape
|
|
|
897 sequences, etc. */
|
|
|
898 unsigned int eof:1;
|
|
|
899
|
|
|
900 /* CH holds a partially built-up character. This is really part of the
|
|
|
901 state-dependent data and should be moved there. */
|
|
|
902 unsigned int ch;
|
|
|
903
|
|
|
904 /* Coding-system-specific data holding extra state about the
|
|
|
905 conversion. Logically a struct TYPE_coding_stream; a pointer
|
|
800
|
906 to such a struct, with (when ERROR_CHECK_TYPES is defined)
|
|
771
|
907 error-checking that this is really a structure of that type
|
|
|
908 (checking the corresponding coding system type) can be retrieved using
|
|
|
909 CODING_STREAM_TYPE_DATA(). Allocated at the same time that
|
|
|
910 CODESYS is set (which may occur at any time, even multiple times,
|
|
|
911 during the lifetime of the stream). The size comes from
|
|
|
912 methods->coding_data_size. */
|
|
|
913 void *data;
|
|
|
914
|
|
|
915 enum encode_decode direction;
|
|
|
916
|
|
800
|
917 /* If set, don't close the stream at the other end when being closed. */
|
|
|
918 unsigned int no_close_other:1;
|
|
802
|
919 /* If set, read only one byte at a time from other end to avoid any
|
|
|
920 possible blocking. */
|
|
|
921 unsigned int one_byte_at_a_time:1;
|
|
814
|
922 /* If set, and we're a read stream, we init char mode on ourselves as
|
|
|
923 necessary to prevent the caller from getting partial characters. (the
|
|
|
924 default) */
|
|
|
925 unsigned int set_char_mode_on_us_when_reading:1;
|
|
800
|
926
|
|
771
|
927 /* #### Temporary test */
|
|
|
928 unsigned int finalized:1;
|
|
|
929 };
|
|
|
930
|
|
|
931 #define CODING_STREAM_DATA(stream) LSTREAM_TYPE_DATA (stream, coding)
|
|
|
932
|
|
800
|
933 #ifdef ERROR_CHECK_TYPES
|
|
771
|
934 # define CODING_STREAM_TYPE_DATA(s, type) \
|
|
|
935 error_check_##type##_coding_stream_data (s)
|
|
|
936 #else
|
|
|
937 # define CODING_STREAM_TYPE_DATA(s, type) \
|
|
|
938 ((struct type##_coding_stream *) (s)->data)
|
|
|
939 #endif
|
|
|
940
|
|
|
941 /* C should be a binary character in the range 0 - 255; convert
|
|
|
942 to internal format and add to Dynarr DST. */
|
|
|
943
|
|
428
|
944 #ifdef MULE
|
|
771
|
945
|
|
|
946 #define DECODE_ADD_BINARY_CHAR(c, dst) \
|
|
|
947 do { \
|
|
826
|
948 if (byte_ascii_p (c)) \
|
|
771
|
949 Dynarr_add (dst, c); \
|
|
826
|
950 else if (byte_c1_p (c)) \
|
|
771
|
951 { \
|
|
|
952 Dynarr_add (dst, LEADING_BYTE_CONTROL_1); \
|
|
|
953 Dynarr_add (dst, c + 0x20); \
|
|
|
954 } \
|
|
|
955 else \
|
|
|
956 { \
|
|
|
957 Dynarr_add (dst, LEADING_BYTE_LATIN_ISO8859_1); \
|
|
|
958 Dynarr_add (dst, c); \
|
|
|
959 } \
|
|
|
960 } while (0)
|
|
|
961
|
|
|
962 #else /* not MULE */
|
|
|
963
|
|
|
964 #define DECODE_ADD_BINARY_CHAR(c, dst) \
|
|
|
965 do { \
|
|
|
966 Dynarr_add (dst, c); \
|
|
|
967 } while (0)
|
|
|
968
|
|
|
969 #endif /* MULE */
|
|
|
970
|
|
|
971 #define DECODE_OUTPUT_PARTIAL_CHAR(ch, dst) \
|
|
|
972 do { \
|
|
|
973 if (ch) \
|
|
|
974 { \
|
|
|
975 DECODE_ADD_BINARY_CHAR (ch, dst); \
|
|
|
976 ch = 0; \
|
|
|
977 } \
|
|
|
978 } while (0)
|
|
428
|
979
|
|
|
980 #ifdef MULE
|
|
|
981 /* Convert shift-JIS code (sj1, sj2) into internal string
|
|
|
982 representation (c1, c2). (The leading byte is assumed.) */
|
|
|
983
|
|
771
|
984 #define DECODE_SHIFT_JIS(sj1, sj2, c1, c2) \
|
|
428
|
985 do { \
|
|
|
986 int I1 = sj1, I2 = sj2; \
|
|
|
987 if (I2 >= 0x9f) \
|
|
|
988 c1 = (I1 << 1) - ((I1 >= 0xe0) ? 0xe0 : 0x60), \
|
|
|
989 c2 = I2 + 2; \
|
|
|
990 else \
|
|
|
991 c1 = (I1 << 1) - ((I1 >= 0xe0) ? 0xe1 : 0x61), \
|
|
|
992 c2 = I2 + ((I2 >= 0x7f) ? 0x60 : 0x61); \
|
|
|
993 } while (0)
|
|
|
994
|
|
|
995 /* Convert the internal string representation of a Shift-JIS character
|
|
|
996 (c1, c2) into Shift-JIS code (sj1, sj2). The leading byte is
|
|
|
997 assumed. */
|
|
|
998
|
|
771
|
999 #define ENCODE_SHIFT_JIS(c1, c2, sj1, sj2) \
|
|
428
|
1000 do { \
|
|
|
1001 int I1 = c1, I2 = c2; \
|
|
|
1002 if (I1 & 1) \
|
|
|
1003 sj1 = (I1 >> 1) + ((I1 < 0xdf) ? 0x31 : 0x71), \
|
|
|
1004 sj2 = I2 - ((I2 >= 0xe0) ? 0x60 : 0x61); \
|
|
|
1005 else \
|
|
|
1006 sj1 = (I1 >> 1) + ((I1 < 0xdf) ? 0x30 : 0x70), \
|
|
|
1007 sj2 = I2 - 2; \
|
|
|
1008 } while (0)
|
|
|
1009 #endif /* MULE */
|
|
|
1010
|
|
771
|
1011 DECLARE_CODING_SYSTEM_TYPE (no_conversion);
|
|
|
1012 DECLARE_CODING_SYSTEM_TYPE (convert_eol);
|
|
|
1013 #if 0
|
|
|
1014 DECLARE_CODING_SYSTEM_TYPE (text_file_wrapper);
|
|
|
1015 #endif /* 0 */
|
|
|
1016 DECLARE_CODING_SYSTEM_TYPE (undecided);
|
|
|
1017 DECLARE_CODING_SYSTEM_TYPE (chain);
|
|
|
1018
|
|
|
1019 #ifdef DEBUG_XEMACS
|
|
|
1020 DECLARE_CODING_SYSTEM_TYPE (internal);
|
|
|
1021 #endif
|
|
|
1022
|
|
|
1023 #ifdef MULE
|
|
|
1024 DECLARE_CODING_SYSTEM_TYPE (iso2022);
|
|
|
1025 DECLARE_CODING_SYSTEM_TYPE (ccl);
|
|
|
1026 DECLARE_CODING_SYSTEM_TYPE (shift_jis);
|
|
|
1027 DECLARE_CODING_SYSTEM_TYPE (big5);
|
|
|
1028 #endif
|
|
|
1029
|
|
|
1030 #ifdef HAVE_ZLIB
|
|
|
1031 DECLARE_CODING_SYSTEM_TYPE (gzip);
|
|
|
1032 #endif
|
|
428
|
1033
|
|
771
|
1034 DECLARE_CODING_SYSTEM_TYPE (unicode);
|
|
428
|
1035
|
|
1315
|
1036 #ifdef WIN32_ANY
|
|
771
|
1037 DECLARE_CODING_SYSTEM_TYPE (mswindows_multibyte_to_unicode);
|
|
|
1038 DECLARE_CODING_SYSTEM_TYPE (mswindows_multibyte);
|
|
428
|
1039 #endif
|
|
771
|
1040
|
|
|
1041 Lisp_Object coding_stream_detected_coding_system (Lstream *stream);
|
|
|
1042 Lisp_Object coding_stream_coding_system (Lstream *stream);
|
|
|
1043 void set_coding_stream_coding_system (Lstream *stream,
|
|
|
1044 Lisp_Object codesys);
|
|
|
1045 Lisp_Object detect_coding_stream (Lisp_Object stream);
|
|
867
|
1046 Ichar decode_big5_char (int o1, int o2);
|
|
771
|
1047 void add_entry_to_coding_system_type_list (struct coding_system_methods *m);
|
|
|
1048 Lisp_Object make_internal_coding_system (Lisp_Object existing,
|
|
2367
|
1049 Ascbyte *prefix,
|
|
771
|
1050 Lisp_Object type,
|
|
|
1051 Lisp_Object description,
|
|
|
1052 Lisp_Object props);
|
|
802
|
1053
|
|
814
|
1054 #define LSTREAM_FL_NO_CLOSE_OTHER (1 << 16)
|
|
|
1055 #define LSTREAM_FL_READ_ONE_BYTE_AT_A_TIME (1 << 17)
|
|
|
1056 #define LSTREAM_FL_NO_INIT_CHAR_MODE_WHEN_READING (1 << 18)
|
|
|
1057
|
|
771
|
1058 Lisp_Object make_coding_input_stream (Lstream *stream, Lisp_Object codesys,
|
|
800
|
1059 enum encode_decode direction,
|
|
802
|
1060 int flags);
|
|
771
|
1061 Lisp_Object make_coding_output_stream (Lstream *stream, Lisp_Object codesys,
|
|
800
|
1062 enum encode_decode direction,
|
|
802
|
1063 int flags);
|
|
771
|
1064 void set_detection_results (struct detection_state *st, int detector,
|
|
|
1065 int given);
|
|
428
|
1066
|
|
440
|
1067 #endif /* INCLUDED_file_coding_h_ */
|
|
|
1068
|
