|
771
|
1 /* Header for encoding conversion functions; coding-system object.
|
|
|
2 #### rename me to coding-system.h
|
|
428
|
3 Copyright (C) 1991, 1995 Free Software Foundation, Inc.
|
|
|
4 Copyright (C) 1995 Sun Microsystems, Inc.
|
|
771
|
5 Copyright (C) 2000, 2001 Ben Wing.
|
|
428
|
6
|
|
|
7 This file is part of XEmacs.
|
|
|
8
|
|
|
9 XEmacs is free software; you can redistribute it and/or modify it
|
|
|
10 under the terms of the GNU General Public License as published by the
|
|
|
11 Free Software Foundation; either version 2, or (at your option) any
|
|
|
12 later version.
|
|
|
13
|
|
|
14 XEmacs is distributed in the hope that it will be useful, but WITHOUT
|
|
|
15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
17 for more details.
|
|
|
18
|
|
|
19 You should have received a copy of the GNU General Public License
|
|
|
20 along with XEmacs; see the file COPYING. If not, write to
|
|
|
21 the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
|
22 Boston, MA 02111-1307, USA. */
|
|
|
23
|
|
|
24 /* Synched up with: Mule 2.3. Not in FSF. */
|
|
|
25
|
|
771
|
26 /* Authorship:
|
|
|
27
|
|
|
28 Current primary author: Ben Wing <ben@xemacs.org>
|
|
|
29
|
|
|
30 Written by Ben Wing <ben@xemacs.org> for XEmacs, 1995, loosely based
|
|
|
31 on code written 91.10.09 by K.Handa <handa@etl.go.jp>.
|
|
|
32 Rewritten again 2000-2001 by Ben Wing to support properly
|
|
|
33 abstracted coding systems.
|
|
|
34 September 2001: Finished last part of abstraction, the detection
|
|
|
35 mechanism.
|
|
|
36 */
|
|
428
|
37
|
|
440
|
38 #ifndef INCLUDED_file_coding_h_
|
|
|
39 #define INCLUDED_file_coding_h_
|
|
428
|
40
|
|
771
|
41 /* Capsule description of the different structures, what their purpose is,
|
|
|
42 how they fit together, and where various bits of data are stored.
|
|
|
43
|
|
|
44 A "coding system" is an algorithm for converting data in one format into
|
|
|
45 data in another format. Currently most of the coding systems we have
|
|
|
46 created concern internationalized text, and convert between the XEmacs
|
|
|
47 internal format for multilingual text, and various external
|
|
|
48 representations of such text. However, any such conversion is possible,
|
|
|
49 for example, compressing or uncompressing text using the gzip algorithm.
|
|
|
50 All coding systems provide both encode and decode routines, so that the
|
|
|
51 conversion can go both ways.
|
|
|
52
|
|
|
53 The way we handle this is by dividing the various potential coding
|
|
|
54 systems into types, analogous to classes in C++. Each coding system
|
|
|
55 type encompasses a series of related coding systems that it can
|
|
|
56 implement, and it has properties which control how exactly the encoding
|
|
|
57 works. A particular set of values for each of the properties makes up a
|
|
|
58 "coding system", and specifies one particular encoding. A `struct
|
|
|
59 Lisp_Coding_System' object encapsulates those settings -- its type, the
|
|
|
60 values chosen for all properties of that type, a name for the coding
|
|
|
61 system, some documentation.
|
|
|
62
|
|
|
63 In addition, there are of course methods associated with a coding system
|
|
|
64 type, implementing the encoding, decoding, etc. These are stored in a
|
|
|
65 `struct coding_system_methods' object, one per coding-system type, which
|
|
|
66 contains mostly function pointers. This is retrievable from the
|
|
|
67 coding-system object (i.e. the struct Lisp_Coding_System), which has a
|
|
|
68 pointer to it.
|
|
|
69
|
|
|
70 In order to actually use a coding system to do an encoding or decoding
|
|
|
71 operation, you need to use a coding Lstream.
|
|
|
72
|
|
|
73 Now let's look more at attached data. All coding systems have certain
|
|
|
74 common data fields -- name, type, documentation, etc. -- as well as a
|
|
|
75 bunch more that are defined by the coding system type. To handle this
|
|
|
76 cleanly, each coding system type defines a structure that holds just the
|
|
|
77 fields of data particular to it, and calls it e.g. `struct
|
|
|
78 iso2022_coding_system' for coding system type `iso2022'. When the
|
|
|
79 memory block holding the coding system object is created, it is sized
|
|
|
80 such that it can hold both the struct Lisp_Coding_System and the struct
|
|
|
81 iso2022_coding_system (or whatever) directly following it. (This is a
|
|
|
82 common trick; another possibility is to have a void * pointer in the
|
|
|
83 struct Lisp_Coding_System, which points to another memory block holding
|
|
|
84 the struct iso2022_coding_system.) A macro is provided
|
|
|
85 (CODING_SYSTEM_TYPE_DATA) to retrieve a pointer of the right type to the
|
|
|
86 type-specific data contained within the overall `struct
|
|
|
87 Lisp_Coding_System' block.
|
|
|
88
|
|
|
89 Lstreams, similarly, are objects of type `struct lstream' holding data
|
|
|
90 about the stream operation (how much data has been read or written, any
|
|
|
91 buffered data, any error conditions, etc.), and like coding systems have
|
|
|
92 different types. They have a structure called `Lstream_implementation',
|
|
|
93 one per lstream type, exactly analogous to `struct
|
|
|
94 coding_system_methods'. In addition, they have type-specific data
|
|
|
95 (specifying, e.g., the file number, FILE *, memory location, other
|
|
|
96 lstream, etc. to read the data from or write it to, and for conversion
|
|
|
97 processes, the current state of the process -- are we decoding ASCII or
|
|
|
98 Kanji characters? are we in the middle of a processing an escape
|
|
|
99 sequence? etc.). This type-specific data is stored in a structure
|
|
|
100 named `struct coding_stream'. Just like for coding systems, the
|
|
|
101 type-independent data in the `struct lstream' and the type-dependent
|
|
|
102 data in the `struct coding_stream' are stored together in the same
|
|
|
103 memory block.
|
|
428
|
104
|
|
771
|
105 Now things get a bit tricky. The `struct coding_stream' is
|
|
|
106 type-specific from the point of view of an lstream, but not from the
|
|
|
107 point of view of a coding system. It contains only general data about
|
|
|
108 the conversion process, e.g. the name of the coding system used for
|
|
|
109 conversion, the lstream that we take data from or write it to (depending
|
|
|
110 on whether this was created as a read stream or a write stream), a
|
|
|
111 buffer to hold extra data we retrieved but can't send on yet, some
|
|
|
112 flags, etc. It also needs some data specific to the particular coding
|
|
|
113 system and thus to the particular operation going on. This data is held
|
|
|
114 in a structure named (e.g.) `struct iso2022_coding_stream', and it's
|
|
|
115 held in a separate memory block and pointed to by the generic `struct
|
|
|
116 coding_stream'. It's not glommed into a single memory block both
|
|
|
117 because that would require making changes to the generic lstream code
|
|
|
118 and more importantly because the coding system used in a particular
|
|
|
119 coding lstream can be changed at any point during the lifetime of the
|
|
|
120 lstream, and possibly multiple times. (For example, it can be set using
|
|
|
121 the Lisp primitives `set-process-input-coding-system' and
|
|
|
122 `set-console-tty-input-coding-system', as well as getting set when a
|
|
|
123 conversion operation was started with coding system `undecided' and the
|
|
|
124 correct coding system was then detected.)
|
|
428
|
125
|
|
771
|
126 IMPORTANT NOTE: There are at least two ancillary data structures
|
|
|
127 associated with a coding system type. (There may also be detection data;
|
|
|
128 see elsewhere.) It's important, when writing a coding system type, to
|
|
|
129 keep straight which type of data goes where. In particular, `struct
|
|
|
130 foo_coding_system' is attached to the coding system object itself. This
|
|
|
131 is a permanent object and there's only one per coding system. It's
|
|
|
132 created once, usually at init time, and never destroyed. So, `struct
|
|
|
133 foo_coding_system' should in general not contain dynamic data! (Just
|
|
|
134 data describing the properties of the coding system.) In particular,
|
|
|
135 *NO* data about any conversion in progress. There may be many
|
|
|
136 conversions going on simultaneously using a particular coding system,
|
|
|
137 and by storing conversion data in the coding system, these conversions
|
|
|
138 will overwrite each other's data.
|
|
|
139
|
|
|
140 Instead, use the lstream object, whose purpose is to encapsulate a
|
|
|
141 particular conversion and all associated data. From the lstream object,
|
|
|
142 you can get the struct coding_stream using something like
|
|
|
143
|
|
|
144 struct coding_stream *str = LSTREAM_TYPE_DATA (lstr, coding);
|
|
|
145
|
|
|
146 But usually this structure is already passed to you as one of the
|
|
|
147 parameters of the method being invoked.
|
|
|
148
|
|
|
149 From the struct coding_stream, you can retrieve the
|
|
|
150 coding-system-type-specific data using something like
|
|
|
151
|
|
|
152 struct foo_coding_stream *data = CODING_STREAM_TYPE_DATA (str, foo);
|
|
|
153
|
|
|
154 Then, use this structure to hold all data relevant to the particular
|
|
|
155 conversion being done.
|
|
|
156
|
|
|
157 Initialize this structure whenever init_coding_stream_method is called
|
|
|
158 (this may happen more than once), and finalize it (free resources, etc.)
|
|
|
159 when finalize_coding_stream_method is called.
|
|
|
160 */
|
|
|
161
|
|
|
162 struct coding_stream;
|
|
|
163 struct detection_state;
|
|
|
164
|
|
|
165 extern const struct struct_description coding_system_methods_description;
|
|
|
166
|
|
|
167 struct coding_system_methods;
|
|
|
168
|
|
|
169 enum source_sink_type
|
|
428
|
170 {
|
|
771
|
171 DECODES_CHARACTER_TO_BYTE,
|
|
|
172 DECODES_BYTE_TO_BYTE,
|
|
|
173 DECODES_BYTE_TO_CHARACTER,
|
|
|
174 DECODES_CHARACTER_TO_CHARACTER
|
|
428
|
175 };
|
|
|
176
|
|
|
177 enum eol_type
|
|
|
178 {
|
|
|
179 EOL_LF,
|
|
|
180 EOL_CRLF,
|
|
771
|
181 EOL_CR,
|
|
|
182 EOL_AUTODETECT,
|
|
428
|
183 };
|
|
|
184
|
|
|
185 struct Lisp_Coding_System
|
|
|
186 {
|
|
|
187 struct lcrecord_header header;
|
|
771
|
188 struct coding_system_methods *methods;
|
|
428
|
189
|
|
771
|
190 /* Name and description of this coding system. The description
|
|
|
191 should be suitable for a menu entry. */
|
|
440
|
192 Lisp_Object name;
|
|
771
|
193 Lisp_Object description;
|
|
428
|
194
|
|
|
195 /* Mnemonic string displayed in the modeline when this coding
|
|
|
196 system is active for a particular buffer. */
|
|
|
197 Lisp_Object mnemonic;
|
|
|
198
|
|
771
|
199 /* Long documentation on the coding system. */
|
|
|
200 Lisp_Object documentation;
|
|
|
201 /* Functions to handle additional conversion after reading or before
|
|
|
202 writing. #### This mechanism should be replaced by the ability to
|
|
|
203 simply create new coding system types. */
|
|
440
|
204 Lisp_Object post_read_conversion;
|
|
|
205 Lisp_Object pre_write_conversion;
|
|
428
|
206
|
|
771
|
207 /* If this coding system is not of the correct type for text file
|
|
|
208 conversion (i.e. decodes byte->char), we wrap it with appropriate
|
|
|
209 char<->byte converters. This is created dynamically, when it's
|
|
|
210 needed, and cached here. */
|
|
|
211 Lisp_Object text_file_wrapper;
|
|
|
212
|
|
|
213 /* If true, this is an internal coding system, which will not show up in
|
|
|
214 coding-system-list unless a special parameter is given to it. */
|
|
|
215 int internal_p;
|
|
|
216
|
|
|
217 /* ------------------------ junk to handle EOL -------------------------
|
|
|
218 I had hoped that we could handle this without lots of special-case
|
|
|
219 code, but it appears not to be the case if we want to maintain
|
|
|
220 compatibility with the existing way. However, at least with the way
|
|
|
221 we do things now, we avoid EOL junk in most of the coding system
|
|
|
222 methods themselves, or in the decode/encode functions. The EOL
|
|
|
223 special-case code is limited to coding-system creation and to the
|
|
|
224 convert-eol and undecided coding system types. */
|
|
|
225
|
|
|
226 /* If this coding system wants autodetection of the EOL type, then at the
|
|
|
227 appropriate time we wrap this coding system with
|
|
|
228 convert-eol-autodetect. (We do NOT do this at creation time because
|
|
|
229 then we end up with multiple convert-eols wrapped into the final
|
|
|
230 result -- esp. with autodetection using `undecided' -- leading to a
|
|
|
231 big mess.) We cache the wrapped coding system here. */
|
|
|
232 Lisp_Object auto_eol_wrapper;
|
|
|
233
|
|
|
234 /* Eol type requested by user. */
|
|
|
235 enum eol_type eol_type;
|
|
428
|
236
|
|
|
237 /* Subsidiary coding systems that specify a particular type of EOL
|
|
|
238 marking, rather than autodetecting it. These will only be non-nil
|
|
771
|
239 if (eol_type == EOL_AUTODETECT). These are chains. */
|
|
|
240 Lisp_Object eol[3];
|
|
|
241 /* If this coding system is a subsidiary, this element points back to its
|
|
|
242 parent. */
|
|
|
243 Lisp_Object subsidiary_parent;
|
|
428
|
244
|
|
771
|
245 /* At decoding or encoding time, we use the following coding system, if
|
|
|
246 it exists, in place of the coding system object. This is how we
|
|
|
247 handle coding systems with EOL types of CRLF or CR. Formerly, we did
|
|
|
248 the canonicalization at creation time, returning a chain in place of
|
|
|
249 the original coding system; but that interferes with
|
|
|
250 `coding-system-property' and causes other complications. CANONICAL is
|
|
|
251 used when determining the end types of a coding system.
|
|
|
252 canonicalize-after-coding also consults CANONICAL (it has to, because
|
|
|
253 the data in the lstream is based on CANONICAL, not on the original
|
|
|
254 coding system). */
|
|
|
255 Lisp_Object canonical;
|
|
|
256
|
|
|
257 /* type-specific extra data attached to a coding_system */
|
|
|
258 char data[1];
|
|
428
|
259 };
|
|
|
260 typedef struct Lisp_Coding_System Lisp_Coding_System;
|
|
|
261
|
|
440
|
262 DECLARE_LRECORD (coding_system, Lisp_Coding_System);
|
|
|
263 #define XCODING_SYSTEM(x) XRECORD (x, coding_system, Lisp_Coding_System)
|
|
428
|
264 #define XSETCODING_SYSTEM(x, p) XSETRECORD (x, p, coding_system)
|
|
617
|
265 #define wrap_coding_system(p) wrap_record (p, coding_system)
|
|
428
|
266 #define CODING_SYSTEMP(x) RECORDP (x, coding_system)
|
|
|
267 #define CHECK_CODING_SYSTEM(x) CHECK_RECORD (x, coding_system)
|
|
|
268 #define CONCHECK_CODING_SYSTEM(x) CONCHECK_RECORD (x, coding_system)
|
|
|
269
|
|
771
|
270 struct coding_system_methods
|
|
|
271 {
|
|
|
272 Lisp_Object type;
|
|
|
273 Lisp_Object predicate_symbol;
|
|
|
274
|
|
|
275 /* Implementation specific methods: */
|
|
|
276
|
|
|
277 /* Init method: Initialize coding-system data. Optional. */
|
|
|
278 void (*init_method) (Lisp_Object coding_system);
|
|
|
279
|
|
|
280 /* Mark method: Mark any Lisp objects in the type-specific data
|
|
|
281 attached to the coding-system object. Optional. */
|
|
|
282 void (*mark_method) (Lisp_Object coding_system);
|
|
|
283
|
|
|
284 /* Print method: Print the type-specific properties of this coding
|
|
|
285 system, as part of `print'-ing the object. If this method is defined
|
|
|
286 and prints anything, it should print a space as the first thing it
|
|
|
287 does. Optional. */
|
|
|
288 void (*print_method) (Lisp_Object cs, Lisp_Object printcharfun,
|
|
|
289 int escapeflag);
|
|
|
290
|
|
|
291 /* Canonicalize method: Convert this coding system to another one; called
|
|
|
292 once, at creation time, after all properties have been parsed. The
|
|
|
293 returned value should be a coding system created with
|
|
|
294 make_internal_coding_system() (passing the existing coding system as the
|
|
|
295 first argument), and will become the coding system returned by
|
|
|
296 `make-coding-system'. Optional.
|
|
|
297
|
|
|
298 NOTE: There are *three* different uses of "canonical" or "canonicalize"
|
|
|
299 w.r.t. coding systems, and it's important to keep them straight.
|
|
|
300
|
|
|
301 1. The canonicalize method. Used to specify a different coding
|
|
|
302 system, used when doing conversions, in place of the actual coding
|
|
|
303 system itself. Stored in the CANONICAL field of a coding system.
|
|
|
304
|
|
|
305 2. The canonicalize-after-coding method. Used to return the encoding
|
|
|
306 that was "actually" used to decode some text, such that this
|
|
|
307 particular encoding can be used to encode the text again with the
|
|
|
308 expectation that the result will be the same as the original encoding.
|
|
|
309 Particularly important with auto-detecting coding systems.
|
|
|
310
|
|
|
311 3. From the perspective of aliases, a "canonical" coding system is one
|
|
|
312 that's not an alias to some other coding system, and "canonicalization"
|
|
|
313 is the process of traversing the alias pointers to find the canonical
|
|
|
314 coding system that's equivalent to the alias.
|
|
|
315 */
|
|
|
316 Lisp_Object (*canonicalize_method) (Lisp_Object coding_system);
|
|
|
317
|
|
|
318 /* Canonicalize after coding method: Convert this coding system to
|
|
|
319 another one, after coding (usually decoding) has finished. This is
|
|
|
320 meant to be used by auto-detecting coding systems, which should return
|
|
|
321 the actually detected coding system. Optional. */
|
|
|
322 Lisp_Object (*canonicalize_after_coding_method)
|
|
|
323 (struct coding_stream *str);
|
|
|
324
|
|
|
325 /* Convert method: Decode or encode the data in SRC of size N, writing
|
|
|
326 the results into the Dynarr DST. If the conversion_end_type method
|
|
|
327 indicates that the source is characters (as opposed to bytes), you are
|
|
|
328 guaranteed to get only whole characters in the data in SRC/N. STR, a
|
|
|
329 struct coding_stream, stores all necessary state and other info about
|
|
|
330 the conversion. Coding-specific state (struct TYPE_coding_stream) can
|
|
|
331 be retrieved from STR using CODING_STREAM_TYPE_DATA(). Return value
|
|
|
332 indicates the number of bytes of the *INPUT* that were converted (not
|
|
|
333 the number of bytes written to the Dynarr!). This can be less than
|
|
|
334 the total amount of input passed in; if so, the remainder is
|
|
|
335 considered "rejected" and will appear again at the beginning of the
|
|
|
336 data passed in the next time the convert method is called. When EOF
|
|
|
337 is returned on the other end and there's no more data, the convert
|
|
|
338 method will be called one last time, STR->eof set and the passed-in
|
|
|
339 data will consist only of any rejected data from the previous
|
|
|
340 call. (At this point, file handles and similar resources can be
|
|
|
341 closed, but do NOT arbitrarily free data structures in the
|
|
|
342 type-specific data, because there are operations that can be done on
|
|
|
343 closed streams to query the results of the processing -- specifically,
|
|
|
344 for coding streams, there's the canonicalize_after_coding() method.)
|
|
|
345 Required. */
|
|
|
346 Bytecount (*convert_method) (struct coding_stream *str,
|
|
|
347 const unsigned char *src,
|
|
|
348 unsigned_char_dynarr *dst, Bytecount n);
|
|
|
349
|
|
|
350 /* Coding mark method: Mark any Lisp objects in the type-specific data
|
|
|
351 attached to `struct coding_stream'. Optional. */
|
|
|
352 void (*mark_coding_stream_method) (struct coding_stream *str);
|
|
|
353
|
|
|
354 /* Init coding stream method: Initialize the type-specific data attached
|
|
|
355 to the coding stream (i.e. in struct TYPE_coding_stream), when the
|
|
|
356 coding stream is opened. The type-specific data will be zeroed out.
|
|
|
357 Optional. */
|
|
|
358 void (*init_coding_stream_method) (struct coding_stream *str);
|
|
|
359
|
|
|
360 /* Rewind coding stream method: Reset any necessary type-specific data as
|
|
|
361 a result of the stream being rewound. Optional. */
|
|
|
362 void (*rewind_coding_stream_method) (struct coding_stream *str);
|
|
|
363
|
|
|
364 /* Finalize coding stream method: Clean up the type-specific data
|
|
|
365 attached to the coding stream (i.e. in struct TYPE_coding_stream).
|
|
|
366 Happens when the Lstream is deleted using Lstream_delete() or is
|
|
|
367 garbage-collected. Most streams are deleted after they've been used,
|
|
|
368 so it's less likely (but still possible) that allocated data will
|
|
|
369 stick around until GC time. (File handles can also be closed when EOF
|
|
|
370 is signalled; but some data must stick around after this point, for
|
|
|
371 the benefit of canonicalize_after_coding. See the convert method.)
|
|
|
372 Called only once (NOT called at disksave time). Optional. */
|
|
|
373 void (*finalize_coding_stream_method) (struct coding_stream *str);
|
|
|
374
|
|
|
375 /* Finalize method: Clean up type-specific data (e.g. free allocated
|
|
|
376 data) attached to the coding system (i.e. in struct
|
|
|
377 TYPE_coding_system), when the coding system is about to be garbage
|
|
|
378 collected. (Currently not called.) Called only once (NOT called at
|
|
|
379 disksave time). Optional. */
|
|
|
380 void (*finalize_method) (Lisp_Object codesys);
|
|
|
381
|
|
|
382 /* Conversion end type method: Does this coding system encode bytes ->
|
|
|
383 characters, characters -> characters, bytes -> bytes, or
|
|
|
384 characters -> bytes?. Default is characters -> bytes. Optional. */
|
|
|
385 enum source_sink_type (*conversion_end_type_method) (Lisp_Object codesys);
|
|
|
386
|
|
|
387 /* Putprop method: Set the value of a type-specific property. If
|
|
|
388 the property name is unrecognized, return 0. If the value is disallowed
|
|
|
389 or erroneous, signal an error. Currently called only at creation time.
|
|
|
390 Optional. */
|
|
|
391 int (*putprop_method) (Lisp_Object codesys,
|
|
|
392 Lisp_Object key,
|
|
|
393 Lisp_Object value);
|
|
|
394
|
|
|
395 /* Getprop method: Return the value of a type-specific property. If
|
|
|
396 the property name is unrecognized, return Qunbound. Optional.
|
|
|
397 */
|
|
|
398 Lisp_Object (*getprop_method) (Lisp_Object coding_system,
|
|
|
399 Lisp_Object prop);
|
|
|
400
|
|
|
401 /* These next three are set as part of the call to
|
|
|
402 INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA. */
|
|
|
403
|
|
|
404 /* Description of the extra data (struct foo_coding_system) attached to a
|
|
|
405 coding system, for pdump purposes. NOTE: All offsets must have
|
|
|
406 coding_system_data_offset added to them! */
|
|
|
407 const struct lrecord_description *extra_description;
|
|
|
408 /* size of struct foo_coding_system -- extra data associated with
|
|
|
409 the coding system */
|
|
|
410 int extra_data_size;
|
|
|
411 /* size of struct foo_coding_stream -- extra data associated with the
|
|
|
412 struct coding_stream, needed for each active coding process
|
|
|
413 using this coding system. note that we can have more than one
|
|
|
414 process active at once (simply by creating more than one coding
|
|
|
415 lstream using this coding system), so we can't store this data in
|
|
|
416 the coding system object. */
|
|
|
417 int coding_data_size;
|
|
|
418 };
|
|
|
419
|
|
|
420 /***** Calling a coding-system method *****/
|
|
|
421
|
|
|
422 #define RAW_CODESYSMETH(cs, m) ((cs)->methods->m##_method)
|
|
|
423 #define HAS_CODESYSMETH_P(cs, m) (!!RAW_CODESYSMETH (cs, m))
|
|
|
424 #define CODESYSMETH(cs, m, args) (((cs)->methods->m##_method) args)
|
|
|
425
|
|
|
426 /* Call a void-returning coding-system method, if it exists. */
|
|
|
427 #define MAYBE_CODESYSMETH(cs, m, args) do { \
|
|
|
428 Lisp_Coding_System *maybe_codesysmeth_cs = (cs); \
|
|
|
429 if (HAS_CODESYSMETH_P (maybe_codesysmeth_cs, m)) \
|
|
|
430 CODESYSMETH (maybe_codesysmeth_cs, m, args); \
|
|
|
431 } while (0)
|
|
|
432
|
|
|
433 /* Call a coding-system method, if it exists, or return GIVEN.
|
|
|
434 NOTE: Multiply-evaluates CS. */
|
|
|
435 #define CODESYSMETH_OR_GIVEN(cs, m, args, given) \
|
|
|
436 (HAS_CODESYSMETH_P (cs, m) ? \
|
|
|
437 CODESYSMETH (cs, m, args) : (given))
|
|
|
438
|
|
|
439 #define XCODESYSMETH(cs, m, args) \
|
|
|
440 CODESYSMETH (XCODING_SYSTEM (cs), m, args)
|
|
|
441 #define MAYBE_XCODESYSMETH(cs, m, args) \
|
|
|
442 MAYBE_CODESYSMETH (XCODING_SYSTEM (cs), m, args)
|
|
|
443 #define XCODESYSMETH_OR_GIVEN(cs, m, args, given) \
|
|
|
444 CODESYSMETH_OR_GIVEN (XCODING_SYSTEM (cs), m, args, given)
|
|
|
445
|
|
|
446
|
|
|
447 /***** Defining new coding-system types *****/
|
|
|
448
|
|
|
449 #define coding_system_data_offset (offsetof (Lisp_Coding_System, data))
|
|
|
450 extern const struct lrecord_description coding_system_empty_extra_description[];
|
|
|
451
|
|
|
452 #ifdef ERROR_CHECK_TYPECHECK
|
|
|
453 #define DECLARE_CODING_SYSTEM_TYPE(type) \
|
|
|
454 \
|
|
|
455 extern struct coding_system_methods * type##_coding_system_methods; \
|
|
|
456 INLINE_HEADER struct type##_coding_system * \
|
|
|
457 error_check_##type##_coding_system_data (Lisp_Coding_System *cs); \
|
|
|
458 INLINE_HEADER struct type##_coding_system * \
|
|
|
459 error_check_##type##_coding_system_data (Lisp_Coding_System *cs) \
|
|
|
460 { \
|
|
|
461 assert (CODING_SYSTEM_TYPE_P (cs, type)); \
|
|
|
462 /* Catch accidental use of INITIALIZE_CODING_SYSTEM_TYPE in place \
|
|
|
463 of INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA. */ \
|
|
|
464 assert (cs->methods->extra_data_size > 0); \
|
|
|
465 return (struct type##_coding_system *) cs->data; \
|
|
|
466 } \
|
|
|
467 \
|
|
|
468 INLINE_HEADER struct type##_coding_stream * \
|
|
|
469 error_check_##type##_coding_stream_data (struct coding_stream *s); \
|
|
|
470 INLINE_HEADER struct type##_coding_stream * \
|
|
|
471 error_check_##type##_coding_stream_data (struct coding_stream *s) \
|
|
|
472 { \
|
|
|
473 assert (XCODING_SYSTEM_TYPE_P (s->codesys, type)); \
|
|
|
474 return (struct type##_coding_stream *) s->data; \
|
|
|
475 } \
|
|
|
476 \
|
|
|
477 INLINE_HEADER Lisp_Coding_System * \
|
|
|
478 error_check_##type##_coding_system_type (Lisp_Object obj); \
|
|
|
479 INLINE_HEADER Lisp_Coding_System * \
|
|
|
480 error_check_##type##_coding_system_type (Lisp_Object obj) \
|
|
|
481 { \
|
|
|
482 Lisp_Coding_System *cs = XCODING_SYSTEM (obj); \
|
|
|
483 assert (CODING_SYSTEM_TYPE_P (cs, type)); \
|
|
|
484 return cs; \
|
|
|
485 } \
|
|
|
486 \
|
|
|
487 DECLARE_NOTHING
|
|
|
488 #else
|
|
|
489 #define DECLARE_CODING_SYSTEM_TYPE(type) \
|
|
|
490 extern struct coding_system_methods * type##_coding_system_methods
|
|
|
491 #endif /* ERROR_CHECK_TYPECHECK */
|
|
|
492
|
|
|
493 #define DEFINE_CODING_SYSTEM_TYPE(type) \
|
|
|
494 struct coding_system_methods * type##_coding_system_methods
|
|
|
495
|
|
|
496 #define INITIALIZE_CODING_SYSTEM_TYPE(ty, pred_sym) do { \
|
|
|
497 ty##_coding_system_methods = \
|
|
|
498 xnew_and_zero (struct coding_system_methods); \
|
|
|
499 ty##_coding_system_methods->type = Q##ty; \
|
|
|
500 ty##_coding_system_methods->extra_description = \
|
|
|
501 coding_system_empty_extra_description; \
|
|
|
502 defsymbol_nodump (&ty##_coding_system_methods->predicate_symbol, \
|
|
|
503 pred_sym); \
|
|
|
504 add_entry_to_coding_system_type_list (ty##_coding_system_methods); \
|
|
|
505 dump_add_root_struct_ptr (&ty##_coding_system_methods, \
|
|
|
506 &coding_system_methods_description); \
|
|
|
507 } while (0)
|
|
|
508
|
|
|
509 #define REINITIALIZE_CODING_SYSTEM_TYPE(type) do { \
|
|
|
510 staticpro_nodump (&type##_coding_system_methods->predicate_symbol); \
|
|
|
511 } while (0)
|
|
|
512
|
|
|
513 /* This assumes the existence of two structures:
|
|
|
514
|
|
|
515 struct foo_coding_system (attached to the coding system)
|
|
|
516 struct foo_coding_stream (per coding process, attached to the
|
|
|
517 struct coding_stream)
|
|
|
518 const struct foo_coding_system_description[] (pdump description of
|
|
|
519 struct foo_coding_system)
|
|
|
520
|
|
|
521 NOTE: The description must have coding_system_data_offset added to
|
|
|
522 all offsets in it! For an example of how to do things, see
|
|
|
523 chain_coding_system_description.
|
|
|
524 */
|
|
|
525 #define INITIALIZE_CODING_SYSTEM_TYPE_WITH_DATA(type, pred_sym) \
|
|
|
526 do { \
|
|
|
527 INITIALIZE_CODING_SYSTEM_TYPE (type, pred_sym); \
|
|
|
528 type##_coding_system_methods->extra_data_size = \
|
|
|
529 sizeof (struct type##_coding_system); \
|
|
|
530 type##_coding_system_methods->extra_description = \
|
|
|
531 type##_coding_system_description; \
|
|
|
532 type##_coding_system_methods->coding_data_size = \
|
|
|
533 sizeof (struct type##_coding_stream); \
|
|
|
534 } while (0)
|
|
|
535
|
|
|
536 /* Declare that coding-system-type TYPE has method METH; used in
|
|
|
537 initialization routines */
|
|
|
538 #define CODING_SYSTEM_HAS_METHOD(type, meth) \
|
|
|
539 (type##_coding_system_methods->meth##_method = type##_##meth)
|
|
|
540
|
|
|
541 /***** Macros for accessing coding-system types *****/
|
|
|
542
|
|
|
543 #define CODING_SYSTEM_TYPE_P(cs, type) \
|
|
|
544 ((cs)->methods == type##_coding_system_methods)
|
|
|
545 #define XCODING_SYSTEM_TYPE_P(cs, type) \
|
|
|
546 CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (cs), type)
|
|
|
547
|
|
|
548 #ifdef ERROR_CHECK_TYPECHECK
|
|
|
549 # define CODING_SYSTEM_TYPE_DATA(cs, type) \
|
|
|
550 error_check_##type##_coding_system_data (cs)
|
|
|
551 #else
|
|
|
552 # define CODING_SYSTEM_TYPE_DATA(cs, type) \
|
|
|
553 ((struct type##_coding_system *) \
|
|
|
554 (cs)->data)
|
|
|
555 #endif
|
|
|
556
|
|
|
557 #define XCODING_SYSTEM_TYPE_DATA(cs, type) \
|
|
|
558 CODING_SYSTEM_TYPE_DATA (XCODING_SYSTEM_OF_TYPE (cs, type), type)
|
|
|
559
|
|
|
560 #ifdef ERROR_CHECK_TYPECHECK
|
|
|
561 # define XCODING_SYSTEM_OF_TYPE(x, type) \
|
|
|
562 error_check_##type##_coding_system_type (x)
|
|
|
563 # define XSETCODING_SYSTEM_OF_TYPE(x, p, type) do \
|
|
|
564 { \
|
|
|
565 XSETCODING_SYSTEM (x, p); \
|
|
|
566 assert (CODING_SYSTEM_TYPEP (XCODING_SYSTEM(x), type)); \
|
|
|
567 } while (0)
|
|
|
568 #else
|
|
|
569 # define XCODING_SYSTEM_OF_TYPE(x, type) XCODING_SYSTEM (x)
|
|
|
570 # define XSETCODING_SYSTEM_OF_TYPE(x, p, type) XSETCODING_SYSTEM (x, p)
|
|
|
571 #endif /* ERROR_CHECK_TYPE_CHECK */
|
|
|
572
|
|
|
573 #define CODING_SYSTEM_TYPEP(x, type) \
|
|
|
574 (CODING_SYSTEMP (x) && CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (x), type))
|
|
|
575 #define CHECK_CODING_SYSTEM_OF_TYPE(x, type) do { \
|
|
|
576 CHECK_CODING_SYSTEM (x); \
|
|
|
577 if (!CODING_SYSTEM_TYPE_P (XCODING_SYSTEM (x), type)) \
|
|
|
578 dead_wrong_type_argument \
|
|
|
579 (type##_coding_system_methods->predicate_symbol, x); \
|
|
|
580 } while (0)
|
|
|
581 #define CONCHECK_CODING_SYSTEM_OF_TYPE(x, type) do { \
|
|
|
582 CONCHECK_CODING_SYSTEM (x); \
|
|
|
583 if (!(CODING_SYSTEM_TYPEP (x, type))) \
|
|
|
584 x = wrong_type_argument \
|
|
|
585 (type##_coding_system_methods->predicate_symbol, x); \
|
|
|
586 } while (0)
|
|
|
587
|
|
|
588 #define CODING_SYSTEM_METHODS(codesys) ((codesys)->methods)
|
|
428
|
589 #define CODING_SYSTEM_NAME(codesys) ((codesys)->name)
|
|
771
|
590 #define CODING_SYSTEM_DESCRIPTION(codesys) ((codesys)->description)
|
|
|
591 #define CODING_SYSTEM_TYPE(codesys) ((codesys)->methods->type)
|
|
428
|
592 #define CODING_SYSTEM_MNEMONIC(codesys) ((codesys)->mnemonic)
|
|
771
|
593 #define CODING_SYSTEM_DOCUMENTATION(codesys) ((codesys)->documentation)
|
|
428
|
594 #define CODING_SYSTEM_POST_READ_CONVERSION(codesys) \
|
|
|
595 ((codesys)->post_read_conversion)
|
|
|
596 #define CODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
|
|
|
597 ((codesys)->pre_write_conversion)
|
|
|
598 #define CODING_SYSTEM_EOL_TYPE(codesys) ((codesys)->eol_type)
|
|
771
|
599 #define CODING_SYSTEM_EOL_LF(codesys) ((codesys)->eol[EOL_LF])
|
|
|
600 #define CODING_SYSTEM_EOL_CRLF(codesys) ((codesys)->eol[EOL_CRLF])
|
|
|
601 #define CODING_SYSTEM_EOL_CR(codesys) ((codesys)->eol[EOL_CR])
|
|
|
602 #define CODING_SYSTEM_TEXT_FILE_WRAPPER(codesys) ((codesys)->text_file_wrapper)
|
|
|
603 #define CODING_SYSTEM_AUTO_EOL_WRAPPER(codesys) ((codesys)->auto_eol_wrapper)
|
|
|
604 #define CODING_SYSTEM_SUBSIDIARY_PARENT(codesys) ((codesys)->subsidiary_parent)
|
|
|
605 #define CODING_SYSTEM_CANONICAL(codesys) ((codesys)->canonical)
|
|
428
|
606
|
|
771
|
607 #define CODING_SYSTEM_CHAIN_CHAIN(codesys) \
|
|
|
608 (CODING_SYSTEM_TYPE_DATA (codesys, chain)->chain)
|
|
|
609 #define CODING_SYSTEM_CHAIN_COUNT(codesys) \
|
|
|
610 (CODING_SYSTEM_TYPE_DATA (codesys, chain)->count)
|
|
|
611 #define CODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING(codesys) \
|
|
|
612 (CODING_SYSTEM_TYPE_DATA (codesys, chain)->canonicalize_after_coding)
|
|
428
|
613
|
|
771
|
614 #define XCODING_SYSTEM_METHODS(codesys) \
|
|
|
615 CODING_SYSTEM_METHODS (XCODING_SYSTEM (codesys))
|
|
428
|
616 #define XCODING_SYSTEM_NAME(codesys) \
|
|
|
617 CODING_SYSTEM_NAME (XCODING_SYSTEM (codesys))
|
|
771
|
618 #define XCODING_SYSTEM_DESCRIPTION(codesys) \
|
|
|
619 CODING_SYSTEM_DESCRIPTION (XCODING_SYSTEM (codesys))
|
|
428
|
620 #define XCODING_SYSTEM_TYPE(codesys) \
|
|
|
621 CODING_SYSTEM_TYPE (XCODING_SYSTEM (codesys))
|
|
|
622 #define XCODING_SYSTEM_MNEMONIC(codesys) \
|
|
|
623 CODING_SYSTEM_MNEMONIC (XCODING_SYSTEM (codesys))
|
|
771
|
624 #define XCODING_SYSTEM_DOCUMENTATION(codesys) \
|
|
|
625 CODING_SYSTEM_DOCUMENTATION (XCODING_SYSTEM (codesys))
|
|
428
|
626 #define XCODING_SYSTEM_POST_READ_CONVERSION(codesys) \
|
|
|
627 CODING_SYSTEM_POST_READ_CONVERSION (XCODING_SYSTEM (codesys))
|
|
|
628 #define XCODING_SYSTEM_PRE_WRITE_CONVERSION(codesys) \
|
|
|
629 CODING_SYSTEM_PRE_WRITE_CONVERSION (XCODING_SYSTEM (codesys))
|
|
|
630 #define XCODING_SYSTEM_EOL_TYPE(codesys) \
|
|
|
631 CODING_SYSTEM_EOL_TYPE (XCODING_SYSTEM (codesys))
|
|
|
632 #define XCODING_SYSTEM_EOL_LF(codesys) \
|
|
|
633 CODING_SYSTEM_EOL_LF (XCODING_SYSTEM (codesys))
|
|
|
634 #define XCODING_SYSTEM_EOL_CRLF(codesys) \
|
|
|
635 CODING_SYSTEM_EOL_CRLF (XCODING_SYSTEM (codesys))
|
|
|
636 #define XCODING_SYSTEM_EOL_CR(codesys) \
|
|
|
637 CODING_SYSTEM_EOL_CR (XCODING_SYSTEM (codesys))
|
|
771
|
638 #define XCODING_SYSTEM_TEXT_FILE_WRAPPER(codesys) \
|
|
|
639 CODING_SYSTEM_TEXT_FILE_WRAPPER (XCODING_SYSTEM (codesys))
|
|
|
640 #define XCODING_SYSTEM_AUTO_EOL_WRAPPER(codesys) \
|
|
|
641 CODING_SYSTEM_AUTO_EOL_WRAPPER (XCODING_SYSTEM (codesys))
|
|
|
642 #define XCODING_SYSTEM_SUBSIDIARY_PARENT(codesys) \
|
|
|
643 CODING_SYSTEM_SUBSIDIARY_PARENT (XCODING_SYSTEM (codesys))
|
|
|
644 #define XCODING_SYSTEM_CANONICAL(codesys) \
|
|
|
645 CODING_SYSTEM_CANONICAL (XCODING_SYSTEM (codesys))
|
|
428
|
646
|
|
771
|
647 #define XCODING_SYSTEM_CHAIN_CHAIN(codesys) \
|
|
|
648 CODING_SYSTEM_CHAIN_CHAIN (XCODING_SYSTEM (codesys))
|
|
|
649 #define XCODING_SYSTEM_CHAIN_COUNT(codesys) \
|
|
|
650 CODING_SYSTEM_CHAIN_COUNT (XCODING_SYSTEM (codesys))
|
|
|
651 #define XCODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING(codesys) \
|
|
|
652 CODING_SYSTEM_CHAIN_CANONICALIZE_AFTER_CODING (XCODING_SYSTEM (codesys))
|
|
428
|
653
|
|
771
|
654 /**************************************************/
|
|
|
655 /* Detection */
|
|
|
656 /**************************************************/
|
|
428
|
657
|
|
771
|
658 #define MAX_DETECTOR_CATEGORIES 256
|
|
|
659 #define MAX_DETECTORS 64
|
|
428
|
660
|
|
771
|
661 #define MAX_BYTES_PROCESSED_FOR_DETECTION 65536
|
|
428
|
662
|
|
771
|
663 struct detection_state
|
|
428
|
664 {
|
|
771
|
665 int seen_non_ascii;
|
|
|
666 Bytecount bytes_seen;
|
|
428
|
667
|
|
771
|
668 char categories[MAX_DETECTOR_CATEGORIES];
|
|
|
669 Bytecount data_offset[MAX_DETECTORS];
|
|
|
670 /* ... more data follows; data_offset[detector_##TYPE] points to
|
|
|
671 the data for that type */
|
|
428
|
672 };
|
|
|
673
|
|
771
|
674 #define DETECTION_STATE_DATA(st, type) \
|
|
|
675 ((struct type##_detector *) \
|
|
|
676 ((char *) (st) + (st)->data_offset[detector_##type]))
|
|
428
|
677
|
|
448
|
678 /* Distinguishable categories of encodings.
|
|
|
679
|
|
|
680 This list determines the initial priority of the categories.
|
|
|
681
|
|
|
682 For better or worse, currently Mule files are encoded in 7-bit ISO 2022.
|
|
|
683 For this reason, under Mule ISO_7 gets highest priority.
|
|
|
684
|
|
|
685 Putting NO_CONVERSION second prevents "binary corruption" in the
|
|
|
686 default case in all but the (presumably) extremely rare case of a
|
|
|
687 binary file which contains redundant escape sequences but no 8-bit
|
|
|
688 characters.
|
|
|
689
|
|
|
690 The remaining priorities are based on perceived "internationalization
|
|
|
691 political correctness." An exception is UCS-4 at the bottom, since
|
|
|
692 basically everything is compatible with UCS-4, but it is likely to
|
|
|
693 be very rare as an external encoding. */
|
|
|
694
|
|
771
|
695 /* Macros to define code of control characters for ISO2022's functions. */
|
|
|
696 /* Used by the detection routines of other coding system types as well. */
|
|
|
697 /* code */ /* function */
|
|
|
698 #define ISO_CODE_LF 0x0A /* line-feed */
|
|
|
699 #define ISO_CODE_CR 0x0D /* carriage-return */
|
|
|
700 #define ISO_CODE_SO 0x0E /* shift-out */
|
|
|
701 #define ISO_CODE_SI 0x0F /* shift-in */
|
|
|
702 #define ISO_CODE_ESC 0x1B /* escape */
|
|
|
703 #define ISO_CODE_DEL 0x7F /* delete */
|
|
|
704 #define ISO_CODE_SS2 0x8E /* single-shift-2 */
|
|
|
705 #define ISO_CODE_SS3 0x8F /* single-shift-3 */
|
|
|
706 #define ISO_CODE_CSI 0x9B /* control-sequence-introduce */
|
|
|
707
|
|
|
708 enum detection_result
|
|
|
709 {
|
|
|
710 /* Basically means a magic cookie was seen indicating this type, or
|
|
|
711 something similar. */
|
|
|
712 DET_NEAR_CERTAINTY = 4,
|
|
|
713 DET_HIGHEST = 4,
|
|
|
714 /* Characteristics seen that are unlikely to be other coding system types
|
|
|
715 -- e.g. ISO-2022 escape sequences, or perhaps a consistent pattern of
|
|
|
716 alternating zero bytes in UTF-16, along with Unicode LF or CRLF
|
|
|
717 sequences at regular intervals. (Zero bytes are unlikely or impossible
|
|
|
718 in most text encodings.) */
|
|
|
719 DET_QUITE_PROBABLE = 3,
|
|
|
720 /* Strong or medium statistical likelihood. At least some
|
|
|
721 characteristics seen that match what's normally found in this encoding
|
|
|
722 -- e.g. in Shift-JIS, a number of two-byte Japanese character
|
|
|
723 sequences in the right range, and nothing out of range; or in Unicode,
|
|
|
724 much higher statistical variance in the odd bytes than in the even
|
|
|
725 bytes, or vice-versa (perhaps the presence of regular EOL sequences
|
|
|
726 would bump this too to DET_QUITE_PROBABLE). This is quite often a
|
|
|
727 statistical test. */
|
|
|
728 DET_SOMEWHAT_LIKELY = 2,
|
|
|
729 /* Weak statistical likelihood. Pretty much any features at all that
|
|
|
730 characterize this encoding, and nothing that rules against it. */
|
|
|
731 DET_SLIGHTLY_LIKELY = 1,
|
|
|
732 /* Default state. Perhaps it indicates pure ASCII or something similarly
|
|
|
733 vague seen in Shift-JIS, or, exactly as the level says, it might mean
|
|
|
734 in a statistical-based detector that the pros and cons are balanced
|
|
|
735 out. This is also the lowest level that will be accepted by the
|
|
|
736 auto-detector without asking the user: If all available detectors
|
|
|
737 report lower levels for all categories with attached coding systems,
|
|
|
738 the user will be shown the results and explicitly prompted for action.
|
|
|
739 The user will also be prompted if this is the highest available level
|
|
|
740 and more than one detector reports the level. (See below about the
|
|
|
741 consequent necessity of an "ASCII" detector, which will return level 1
|
|
|
742 or higher for most plain text files.) */
|
|
|
743 DET_AS_LIKELY_AS_UNLIKELY = 0,
|
|
|
744 /* Some characteristics seen that are unusual for this encoding --
|
|
|
745 e.g. unusual control characters in a plain-text encoding, lots of
|
|
|
746 8-bit characters, or little statistical variance in the odd and even
|
|
|
747 bytes in UTF-16. */
|
|
|
748 DET_SOMEWHAT_UNLIKELY = -1,
|
|
|
749 /* This indicates that there is very little chance the data is in the
|
|
|
750 right format; this is probably the lowest level you can get when
|
|
|
751 presenting random binary data to a text file, because there are no
|
|
|
752 "specific sequences" you can see that would totally rule out
|
|
|
753 recognition. */
|
|
|
754 DET_QUITE_IMPROBABLE = -2,
|
|
|
755 /* An erroneous sequence was seen. */
|
|
|
756 DET_NEARLY_IMPOSSIBLE = -3,
|
|
|
757 DET_LOWEST = 3,
|
|
|
758 };
|
|
|
759
|
|
|
760 extern int coding_detector_count;
|
|
|
761 extern int coding_detector_category_count;
|
|
|
762
|
|
|
763 struct detector_category
|
|
428
|
764 {
|
|
771
|
765 int id;
|
|
|
766 Lisp_Object sym;
|
|
|
767 };
|
|
|
768
|
|
|
769 typedef struct
|
|
|
770 {
|
|
|
771 Dynarr_declare (struct detector_category);
|
|
|
772 } detector_category_dynarr;
|
|
|
773
|
|
|
774 struct detector
|
|
|
775 {
|
|
|
776 int id;
|
|
|
777 detector_category_dynarr *cats;
|
|
|
778 Bytecount data_size;
|
|
|
779 /* Detect method: Required. */
|
|
|
780 void (*detect_method) (struct detection_state *st,
|
|
|
781 const unsigned char *src, Bytecount n);
|
|
|
782 /* Finalize detection state method: Clean up any allocated data in the
|
|
|
783 detection state. Called only once (NOT called at disksave time).
|
|
|
784 Optional. */
|
|
|
785 void (*finalize_detection_state_method) (struct detection_state *st);
|
|
428
|
786 };
|
|
|
787
|
|
771
|
788 /* Lvalue for a particular detection result -- detection state ST,
|
|
|
789 category CAT */
|
|
|
790 #define DET_RESULT(st, cat) ((st)->categories[detector_category_##cat])
|
|
|
791 /* In state ST, set all detection results associated with detector DET to
|
|
|
792 RESULT. */
|
|
|
793 #define SET_DET_RESULTS(st, det, result) \
|
|
|
794 set_detection_results (st, detector_##det, result)
|
|
|
795
|
|
|
796 typedef struct
|
|
|
797 {
|
|
|
798 Dynarr_declare (struct detector);
|
|
|
799 } detector_dynarr;
|
|
|
800
|
|
|
801 extern detector_dynarr *all_coding_detectors;
|
|
|
802
|
|
|
803 #define DEFINE_DETECTOR_CATEGORY(detector, cat) \
|
|
|
804 int detector_category_##cat
|
|
|
805 #define DECLARE_DETECTOR_CATEGORY(detector, cat) \
|
|
|
806 extern int detector_category_##cat
|
|
|
807 #define INITIALIZE_DETECTOR_CATEGORY(detector, cat) \
|
|
|
808 do { \
|
|
|
809 struct detector_category dog; \
|
|
|
810 xzero (dog); \
|
|
|
811 detector_category_##cat = coding_detector_category_count++; \
|
|
|
812 dump_add_opaque_int (&detector_category_##cat); \
|
|
|
813 dog.id = detector_category_##cat; \
|
|
|
814 dog.sym = Q##cat; \
|
|
|
815 Dynarr_add (Dynarr_at (all_coding_detectors, detector_##detector).cats, \
|
|
|
816 dog); \
|
|
|
817 } while (0)
|
|
|
818
|
|
|
819 #define DEFINE_DETECTOR(Detector) \
|
|
|
820 int detector_##Detector
|
|
|
821 #define DECLARE_DETECTOR(Detector) \
|
|
|
822 extern int detector_##Detector
|
|
|
823 #define INITIALIZE_DETECTOR(Detector) \
|
|
|
824 do { \
|
|
|
825 struct detector det; \
|
|
|
826 xzero (det); \
|
|
|
827 detector_##Detector = coding_detector_count++; \
|
|
|
828 dump_add_opaque_int (&detector_##Detector); \
|
|
|
829 det.id = detector_##Detector; \
|
|
|
830 det.cats = Dynarr_new2 (detector_category_dynarr, \
|
|
|
831 struct detector_category); \
|
|
|
832 det.data_size = sizeof (struct Detector##_detector); \
|
|
|
833 Dynarr_add (all_coding_detectors, det); \
|
|
|
834 } while (0)
|
|
|
835 #define DETECTOR_HAS_METHOD(Detector, Meth) \
|
|
|
836 Dynarr_at (all_coding_detectors, detector_##Detector).Meth##_method = \
|
|
|
837 Detector##_##Meth
|
|
|
838
|
|
|
839
|
|
|
840 /**************************************************/
|
|
|
841 /* Decoding/Encoding */
|
|
|
842 /**************************************************/
|
|
|
843
|
|
|
844 /* Is the source (SOURCEP == 1) or sink (SOURCEP == 0) when encoding specified
|
|
|
845 in characters? */
|
|
|
846
|
|
|
847 enum source_or_sink
|
|
|
848 {
|
|
|
849 CODING_SOURCE,
|
|
|
850 CODING_SINK
|
|
|
851 };
|
|
|
852
|
|
|
853 enum encode_decode
|
|
|
854 {
|
|
|
855 CODING_ENCODE,
|
|
|
856 CODING_DECODE
|
|
|
857 };
|
|
|
858
|
|
|
859 /* Data structure attached to an lstream of type `coding',
|
|
|
860 containing values specific to the coding process. Additional
|
|
|
861 data is stored in the DATA field below; the exact form of that data
|
|
|
862 is controlled by the type of the coding system that governs the
|
|
|
863 conversion (field CODESYS). CODESYS may be set at any time
|
|
|
864 throughout the lifetime of the lstream and possibly more than once.
|
|
|
865 See long comment above for more info. */
|
|
|
866
|
|
|
867 struct coding_stream
|
|
|
868 {
|
|
|
869 /* Coding system that governs the conversion. */
|
|
|
870 Lisp_Object codesys;
|
|
|
871 /* Original coding system, pre-canonicalization. */
|
|
|
872 Lisp_Object orig_codesys;
|
|
|
873
|
|
|
874 /* Back pointer to current stream. */
|
|
|
875 Lstream *us;
|
|
|
876
|
|
|
877 /* Stream that we read the unprocessed data from or write the processed
|
|
|
878 data to. */
|
|
|
879 Lstream *other_end;
|
|
|
880
|
|
|
881 /* In order to handle both reading to and writing from a coding stream,
|
|
|
882 we phrase the conversion methods like write methods -- we can
|
|
|
883 implement reading in terms of a write method but not vice-versa,
|
|
|
884 because the write method is forced to take only what it's given but
|
|
|
885 the read method can read more data from the other end if necessary.
|
|
|
886 On the other hand, the write method is free to generate all the data
|
|
|
887 it wants (and just write it to the other end), but the the read method
|
|
|
888 can return only as much as was asked for, so we need to implement our
|
|
|
889 own buffering. */
|
|
|
890
|
|
|
891 /* If we are reading, then we can return only a fixed amount of data, but
|
|
|
892 the converter is free to return as much as it wants, so we direct it
|
|
|
893 to store the data here and lop off chunks as we need them. If we are
|
|
|
894 writing, we use this because the converter takes a Dynarr but we are
|
|
|
895 supposed to write into a fixed buffer. (NOTE: This introduces an extra
|
|
|
896 memory copy.) */
|
|
|
897 unsigned_char_dynarr *convert_to;
|
|
|
898
|
|
|
899 /* The conversion method might reject some of the data -- this typically
|
|
|
900 includes partial characters, partial escape sequences, etc. When
|
|
|
901 writing, we just pass the rejection up to the Lstream module, and it
|
|
|
902 will buffer the data. When reading, however, we need to do the
|
|
|
903 buffering ourselves, and we put it here, combined with newly read
|
|
|
904 data. */
|
|
|
905 unsigned_char_dynarr *convert_from;
|
|
|
906
|
|
|
907 /* If set, this is the last chunk of data being processed. When this is
|
|
|
908 finished, output any necessary terminating control characters, escape
|
|
|
909 sequences, etc. */
|
|
|
910 unsigned int eof:1;
|
|
|
911
|
|
|
912 /* CH holds a partially built-up character. This is really part of the
|
|
|
913 state-dependent data and should be moved there. */
|
|
|
914 unsigned int ch;
|
|
|
915
|
|
|
916 /* Coding-system-specific data holding extra state about the
|
|
|
917 conversion. Logically a struct TYPE_coding_stream; a pointer
|
|
|
918 to such a struct, with (when ERROR_CHECK_TYPECHECK is defined)
|
|
|
919 error-checking that this is really a structure of that type
|
|
|
920 (checking the corresponding coding system type) can be retrieved using
|
|
|
921 CODING_STREAM_TYPE_DATA(). Allocated at the same time that
|
|
|
922 CODESYS is set (which may occur at any time, even multiple times,
|
|
|
923 during the lifetime of the stream). The size comes from
|
|
|
924 methods->coding_data_size. */
|
|
|
925 void *data;
|
|
|
926
|
|
|
927 enum encode_decode direction;
|
|
|
928
|
|
|
929 /* #### Temporary test */
|
|
|
930 unsigned int finalized:1;
|
|
|
931 };
|
|
|
932
|
|
|
933 #define CODING_STREAM_DATA(stream) LSTREAM_TYPE_DATA (stream, coding)
|
|
|
934
|
|
|
935 #ifdef ERROR_CHECK_TYPECHECK
|
|
|
936 # define CODING_STREAM_TYPE_DATA(s, type) \
|
|
|
937 error_check_##type##_coding_stream_data (s)
|
|
|
938 #else
|
|
|
939 # define CODING_STREAM_TYPE_DATA(s, type) \
|
|
|
940 ((struct type##_coding_stream *) (s)->data)
|
|
|
941 #endif
|
|
|
942
|
|
|
943 /* C should be a binary character in the range 0 - 255; convert
|
|
|
944 to internal format and add to Dynarr DST. */
|
|
|
945
|
|
428
|
946 #ifdef MULE
|
|
771
|
947
|
|
|
948 #define DECODE_ADD_BINARY_CHAR(c, dst) \
|
|
|
949 do { \
|
|
|
950 if (BYTE_ASCII_P (c)) \
|
|
|
951 Dynarr_add (dst, c); \
|
|
|
952 else if (BYTE_C1_P (c)) \
|
|
|
953 { \
|
|
|
954 Dynarr_add (dst, LEADING_BYTE_CONTROL_1); \
|
|
|
955 Dynarr_add (dst, c + 0x20); \
|
|
|
956 } \
|
|
|
957 else \
|
|
|
958 { \
|
|
|
959 Dynarr_add (dst, LEADING_BYTE_LATIN_ISO8859_1); \
|
|
|
960 Dynarr_add (dst, c); \
|
|
|
961 } \
|
|
|
962 } while (0)
|
|
|
963
|
|
|
964 #else /* not MULE */
|
|
|
965
|
|
|
966 #define DECODE_ADD_BINARY_CHAR(c, dst) \
|
|
|
967 do { \
|
|
|
968 Dynarr_add (dst, c); \
|
|
|
969 } while (0)
|
|
|
970
|
|
|
971 #endif /* MULE */
|
|
|
972
|
|
|
973 #define DECODE_OUTPUT_PARTIAL_CHAR(ch, dst) \
|
|
|
974 do { \
|
|
|
975 if (ch) \
|
|
|
976 { \
|
|
|
977 DECODE_ADD_BINARY_CHAR (ch, dst); \
|
|
|
978 ch = 0; \
|
|
|
979 } \
|
|
|
980 } while (0)
|
|
428
|
981
|
|
|
982 #ifdef MULE
|
|
|
983 /* Convert shift-JIS code (sj1, sj2) into internal string
|
|
|
984 representation (c1, c2). (The leading byte is assumed.) */
|
|
|
985
|
|
771
|
986 #define DECODE_SHIFT_JIS(sj1, sj2, c1, c2) \
|
|
428
|
987 do { \
|
|
|
988 int I1 = sj1, I2 = sj2; \
|
|
|
989 if (I2 >= 0x9f) \
|
|
|
990 c1 = (I1 << 1) - ((I1 >= 0xe0) ? 0xe0 : 0x60), \
|
|
|
991 c2 = I2 + 2; \
|
|
|
992 else \
|
|
|
993 c1 = (I1 << 1) - ((I1 >= 0xe0) ? 0xe1 : 0x61), \
|
|
|
994 c2 = I2 + ((I2 >= 0x7f) ? 0x60 : 0x61); \
|
|
|
995 } while (0)
|
|
|
996
|
|
|
997 /* Convert the internal string representation of a Shift-JIS character
|
|
|
998 (c1, c2) into Shift-JIS code (sj1, sj2). The leading byte is
|
|
|
999 assumed. */
|
|
|
1000
|
|
771
|
1001 #define ENCODE_SHIFT_JIS(c1, c2, sj1, sj2) \
|
|
428
|
1002 do { \
|
|
|
1003 int I1 = c1, I2 = c2; \
|
|
|
1004 if (I1 & 1) \
|
|
|
1005 sj1 = (I1 >> 1) + ((I1 < 0xdf) ? 0x31 : 0x71), \
|
|
|
1006 sj2 = I2 - ((I2 >= 0xe0) ? 0x60 : 0x61); \
|
|
|
1007 else \
|
|
|
1008 sj1 = (I1 >> 1) + ((I1 < 0xdf) ? 0x30 : 0x70), \
|
|
|
1009 sj2 = I2 - 2; \
|
|
|
1010 } while (0)
|
|
|
1011 #endif /* MULE */
|
|
|
1012
|
|
771
|
1013 DECLARE_CODING_SYSTEM_TYPE (no_conversion);
|
|
|
1014 DECLARE_CODING_SYSTEM_TYPE (convert_eol);
|
|
|
1015 #if 0
|
|
|
1016 DECLARE_CODING_SYSTEM_TYPE (text_file_wrapper);
|
|
|
1017 #endif /* 0 */
|
|
|
1018 DECLARE_CODING_SYSTEM_TYPE (undecided);
|
|
|
1019 DECLARE_CODING_SYSTEM_TYPE (chain);
|
|
|
1020
|
|
|
1021 #ifdef DEBUG_XEMACS
|
|
|
1022 DECLARE_CODING_SYSTEM_TYPE (internal);
|
|
|
1023 #endif
|
|
|
1024
|
|
|
1025 #ifdef MULE
|
|
|
1026 DECLARE_CODING_SYSTEM_TYPE (iso2022);
|
|
|
1027 DECLARE_CODING_SYSTEM_TYPE (ccl);
|
|
|
1028 DECLARE_CODING_SYSTEM_TYPE (shift_jis);
|
|
|
1029 DECLARE_CODING_SYSTEM_TYPE (big5);
|
|
|
1030 #endif
|
|
|
1031
|
|
|
1032 #ifdef HAVE_ZLIB
|
|
|
1033 DECLARE_CODING_SYSTEM_TYPE (gzip);
|
|
|
1034 #endif
|
|
428
|
1035
|
|
771
|
1036 DECLARE_CODING_SYSTEM_TYPE (unicode);
|
|
428
|
1037
|
|
771
|
1038 #ifdef HAVE_WIN32_CODING_SYSTEMS
|
|
|
1039 DECLARE_CODING_SYSTEM_TYPE (mswindows_multibyte_to_unicode);
|
|
|
1040 DECLARE_CODING_SYSTEM_TYPE (mswindows_multibyte);
|
|
428
|
1041 #endif
|
|
771
|
1042
|
|
|
1043 Lisp_Object coding_stream_detected_coding_system (Lstream *stream);
|
|
|
1044 Lisp_Object coding_stream_coding_system (Lstream *stream);
|
|
|
1045 void set_coding_stream_coding_system (Lstream *stream,
|
|
|
1046 Lisp_Object codesys);
|
|
|
1047 Lisp_Object detect_coding_stream (Lisp_Object stream);
|
|
|
1048 Emchar decode_big5_char (int o1, int o2);
|
|
|
1049 void add_entry_to_coding_system_type_list (struct coding_system_methods *m);
|
|
|
1050 Lisp_Object make_internal_coding_system (Lisp_Object existing,
|
|
|
1051 Char_ASCII *prefix,
|
|
|
1052 Lisp_Object type,
|
|
|
1053 Lisp_Object description,
|
|
|
1054 Lisp_Object props);
|
|
|
1055 Lisp_Object make_coding_input_stream (Lstream *stream, Lisp_Object codesys,
|
|
|
1056 enum encode_decode direction);
|
|
|
1057 Lisp_Object make_coding_output_stream (Lstream *stream, Lisp_Object codesys,
|
|
|
1058 enum encode_decode direction);
|
|
|
1059 void set_detection_results (struct detection_state *st, int detector,
|
|
|
1060 int given);
|
|
428
|
1061
|
|
440
|
1062 #endif /* INCLUDED_file_coding_h_ */
|
|
|
1063
|
