|
70
|
1 /* Header for multilingual functions.
|
|
|
2 Copyright (C) 1992, 1995 Free Software Foundation, Inc.
|
|
|
3 Copyright (C) 1995 Sun Microsystems, Inc.
|
|
|
4
|
|
|
5 This file is part of XEmacs.
|
|
|
6
|
|
|
7 XEmacs is free software; you can redistribute it and/or modify it
|
|
|
8 under the terms of the GNU General Public License as published by the
|
|
|
9 Free Software Foundation; either version 2, or (at your option) any
|
|
|
10 later version.
|
|
|
11
|
|
|
12 XEmacs is distributed in the hope that it will be useful, but WITHOUT
|
|
|
13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
|
15 for more details.
|
|
|
16
|
|
|
17 You should have received a copy of the GNU General Public License
|
|
|
18 along with XEmacs; see the file COPYING. If not, write to
|
|
|
19 the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
|
20 Boston, MA 02111-1307, USA. */
|
|
|
21
|
|
|
22 /* Synched up with: Mule 2.3. Not in FSF. */
|
|
|
23
|
|
|
24 /* Rewritten by Ben Wing <wing@666.com>. */
|
|
|
25
|
|
|
26 #ifndef _XEMACS_MULE_CHARSET_H
|
|
|
27 #define _XEMACS_MULE_CHARSET_H
|
|
|
28
|
|
|
29 /*
|
|
|
30 1. Character Sets
|
|
|
31 =================
|
|
|
32
|
|
|
33 A character set (or "charset") is an ordered set of characters.
|
|
|
34 A particular character in a charset is indexed using one or
|
|
|
35 more "position codes", which are non-negative integers.
|
|
|
36 The number of position codes needed to identify a particular
|
|
|
37 character in a charset is called the "dimension" of the
|
|
|
38 charset. In XEmacs/Mule, all charsets have 1 or 2 dimensions,
|
|
|
39 and the size of all charsets (except for a few special cases)
|
|
|
40 is either 94, 96, 94 by 94, or 96 by 96. The range of
|
|
|
41 position codes used to index characters from any of these
|
|
|
42 types of character sets is as follows:
|
|
|
43
|
|
|
44 Charset type Position code 1 Position code 2
|
|
|
45 ------------------------------------------------------------
|
|
|
46 94 33 - 126 N/A
|
|
|
47 96 32 - 127 N/A
|
|
|
48 94x94 33 - 126 33 - 126
|
|
|
49 96x96 32 - 127 32 - 127
|
|
|
50
|
|
|
51 Note that in the above cases position codes do not start at
|
|
|
52 an expected value such as 0 or 1. The reason for this will
|
|
|
53 become clear later.
|
|
|
54
|
|
|
55 For example, Latin-1 is a 96-character charset, and JISX0208
|
|
|
56 (the Japanese national character set) is a 94x94-character
|
|
|
57 charset.
|
|
|
58
|
|
|
59 [Note that, although the ranges above define the *valid*
|
|
|
60 position codes for a charset, some of the slots in a particular
|
|
|
61 charset may in fact be empty. This is the case for JISX0208,
|
|
|
62 for example, where (e.g.) all the slots whose first
|
|
|
63 position code is in the range 118 - 127 are empty.]
|
|
|
64
|
|
|
65 There are three charsets that do not follow the above rules.
|
|
|
66 All of them have one dimension, and have ranges of position
|
|
|
67 codes as follows:
|
|
|
68
|
|
|
69 Charset name Position code 1
|
|
|
70 ------------------------------------
|
|
|
71 ASCII 0 - 127
|
|
|
72 Control-1 0 - 31
|
|
|
73 Composite 0 - some large number
|
|
|
74
|
|
|
75 (The upper bound of the position code for composite characters
|
|
|
76 has not yet been determined, but it will probably be at
|
|
|
77 least 16,383).
|
|
|
78
|
|
|
79 ASCII is the union of two subsidiary character sets:
|
|
|
80 Printing-ASCII (the printing ASCII character set,
|
|
|
81 consisting of position codes 33 - 126, like for a standard
|
|
|
82 94-character charset) and Control-ASCII (the non-printing
|
|
|
83 characters that would appear in a binary file with codes 0
|
|
|
84 - 32 and 127).
|
|
|
85
|
|
|
86 Control-1 contains the non-printing characters that would
|
|
|
87 appear in a binary file with codes 128 - 159.
|
|
|
88
|
|
|
89 Composite contains characters that are generated by
|
|
|
90 overstriking one or more characters from other charsets.
|
|
|
91
|
|
|
92 Note that some characters in ASCII, and all characters
|
|
|
93 in Control-1, are "control" (non-printing) characters.
|
|
|
94 These have no printed representation but instead control
|
|
|
95 some other function of the printing (e.g. TAB or 8 moves
|
|
|
96 the current character position to the next tab stop).
|
|
|
97 All other characters in all charsets are "graphic"
|
|
|
98 (printing) characters.
|
|
|
99
|
|
|
100 When a binary file is read in, the bytes in the file are
|
|
|
101 assigned to character sets as follows:
|
|
|
102
|
|
|
103 Bytes Character set Range
|
|
|
104 --------------------------------------------------
|
|
|
105 0 - 127 ASCII 0 - 127
|
|
|
106 128 - 159 Control-1 0 - 31
|
|
|
107 160 - 255 Latin-1 32 - 127
|
|
|
108
|
|
|
109 This is a bit ad-hoc but gets the job done.
|
|
|
110
|
|
|
111 2. Encodings
|
|
|
112 ============
|
|
|
113
|
|
|
114 An "encoding" is a way of numerically representing
|
|
|
115 characters from one or more character sets. If an encoding
|
|
|
116 only encompasses one character set, then the position codes
|
|
|
117 for the characters in that character set could be used
|
|
|
118 directly. This is not possible, however, if more than one
|
|
|
119 character set is to be used in the encoding.
|
|
|
120
|
|
|
121 For example, the conversion detailed above between bytes in
|
|
|
122 a binary file and characters is effectively an encoding
|
|
|
123 that encompasses the three character sets ASCII, Control-1,
|
|
|
124 and Latin-1 in a stream of 8-bit bytes.
|
|
|
125
|
|
|
126 Thus, an encoding can be viewed as a way of encoding
|
|
|
127 characters from a specified group of character sets using a
|
|
|
128 stream of bytes, each of which contains a fixed number of
|
|
|
129 bits (but not necessarily 8, as in the common usage of
|
|
|
130 "byte").
|
|
|
131
|
|
|
132 Here are descriptions of a couple of common
|
|
|
133 encodings:
|
|
|
134
|
|
|
135
|
|
|
136 A. Japanese EUC (Extended Unix Code)
|
|
|
137
|
|
|
138 This encompasses the character sets:
|
|
|
139 - Printing-ASCII,
|
|
|
140 - Japanese-JISX0201-Kana (half-width katakana, the right half of JISX0201).
|
|
|
141 - Japanese-JISX0208
|
|
|
142 - Japanese-JISX0212
|
|
|
143 It uses 8-bit bytes.
|
|
|
144
|
|
|
145 Note that Printing-ASCII and Japanese-JISX0201-Kana are 94-character
|
|
|
146 charsets, while Japanese-JISX0208 is a 94x94-character charset.
|
|
|
147
|
|
|
148 The encoding is as follows:
|
|
|
149
|
|
|
150 Character set Representation (PC == position-code)
|
|
|
151 ------------- --------------
|
|
|
152 Printing-ASCII PC-1
|
|
|
153 Japanese-JISX0208 PC-1 + 0x80 / PC-2 + 0x80
|
|
|
154 Japanese-JISX0201-Kana 0x8E / PC-1 + 0x80
|
|
|
155
|
|
|
156
|
|
|
157 B. JIS7
|
|
|
158
|
|
|
159 This encompasses the character sets:
|
|
|
160 - Printing-ASCII
|
|
|
161 - Japanese-JISX0201-Roman (the left half of JISX0201; this
|
|
|
162 character set is very similar to Printing-ASCII and is a
|
|
|
163 94-character charset)
|
|
|
164 - Japanese-JISX0208
|
|
|
165 - Japanese-JISX0201-Kana.
|
|
|
166 It uses 7-bit bytes.
|
|
|
167
|
|
|
168 Unlike Japanese EUC, this is a "modal" encoding, which
|
|
|
169 means that there are multiple states that the encoding can
|
|
|
170 be in, which affect how the bytes are to be interpreted.
|
|
|
171 Special sequences of bytes (called "escape sequences")
|
|
|
172 are used to change states.
|
|
|
173
|
|
|
174 The encoding is as follows:
|
|
|
175
|
|
|
176 Character set Representation
|
|
|
177 ------------- --------------
|
|
|
178 Printing-ASCII PC-1
|
|
|
179 Japanese-JISX0201-Roman PC-1
|
|
|
180 Japanese-JISX0201-Kana PC-1
|
|
|
181 Japanese-JISX0208 PC-1 / PC-2
|
|
|
182
|
|
|
183 Escape sequence ASCII equivalent Meaning
|
|
|
184 --------------- ---------------- -------
|
|
|
185 0x1B 0x28 0x42 ESC ( B invoke Printing-ASCII
|
|
|
186 0x1B 0x28 0x4A ESC ( J invoke Japanese-JISX0201-Roman
|
|
|
187 0x1B 0x28 0x49 ESC ( I invoke Japanese-JISX0201-Kana
|
|
|
188 0x1B 0x24 0x42 ESC $ B invoke Japanese-JISX0208
|
|
|
189
|
|
|
190 Initially, Printing-ASCII is invoked.
|
|
|
191
|
|
|
192 3. Internal Mule Encodings
|
|
|
193 ==========================
|
|
|
194
|
|
|
195 In XEmacs/Mule, each character set is assigned a unique number,
|
|
|
196 called a "leading byte". This is used in the encodings of a
|
|
|
197 character. Leading bytes are in the range 0x80 - 0xFF
|
|
|
198 (except for ASCII, which has a leading byte of 0), although
|
|
|
199 some leading bytes are reserved.
|
|
|
200
|
|
|
201 Charsets whose leading byte is in the range 0x80 - 0x9F are
|
|
|
202 called "official" and are used for built-in charsets.
|
|
|
203 Other charsets are called "private" and have leading bytes
|
|
|
204 in the range 0xA0 - 0xFF; these are user-defined charsets.
|
|
|
205
|
|
|
206 More specifically:
|
|
|
207
|
|
|
208 Character set Leading byte
|
|
|
209 ------------- ------------
|
|
|
210 ASCII 0
|
|
|
211 Composite 0x80
|
|
|
212 Dimension-1 Official 0x81 - 0x8D
|
|
|
213 (0x8E is free)
|
|
|
214 Control 0x8F
|
|
|
215 Dimension-2 Official 0x90 - 0x99
|
|
|
216 (0x9A - 0x9D are free;
|
|
|
217 0x9E and 0x9F are reserved)
|
|
|
218 Dimension-1 Private 0xA0 - 0xEF
|
|
|
219 Dimension-2 Private 0xF0 - 0xFF
|
|
|
220
|
|
|
221 There are two internal encodings for characters in XEmacs/Mule.
|
|
|
222 One is called "string encoding" and is an 8-bit encoding that
|
|
|
223 is used for representing characters in a buffer or string.
|
|
|
224 It uses 1 to 4 bytes per character. The other is called
|
|
|
225 "character encoding" and is a 19-bit encoding that is used
|
|
|
226 for representing characters individually in a variable.
|
|
|
227
|
|
|
228 (In the following descriptions, we'll ignore composite
|
|
|
229 characters for the moment. We also give a general (structural)
|
|
|
230 overview first, followed later by the exact details.)
|
|
|
231
|
|
|
232 A. Internal String Encoding
|
|
|
233
|
|
|
234 ASCII characters are encoded using their position code directly.
|
|
|
235 Other characters are encoded using their leading byte followed
|
|
|
236 by their position code(s) with the high bit set. Characters
|
|
|
237 in private character sets have their leading byte prefixed with
|
|
|
238 a "leading byte prefix", which is either 0x9E or 0x9F. (No
|
|
|
239 character sets are ever assigned these leading bytes.) Specifically:
|
|
|
240
|
|
|
241 Character set Encoding (PC == position-code)
|
|
|
242 ------------- -------- (LB == leading-byte)
|
|
|
243 ASCII PC1 |
|
|
|
244 Control-1 LB | PC1 + 0xA0
|
|
|
245 Dimension-1 official LB | PC1 + 0x80
|
|
|
246 Dimension-1 private 0x9E | LB | PC1 + 0x80
|
|
|
247 Dimension-2 official LB | PC1 | PC2 + 0x80
|
|
|
248 Dimension-2 private 0x9F | LB | PC1 + 0x80 | PC2 + 0x80
|
|
|
249
|
|
|
250 The basic characteristic of this encoding is that the first byte
|
|
|
251 of all characters is in the range 0x00 - 0x9F, and the second and
|
|
|
252 following bytes of all characters is in the range 0xA0 - 0xFF.
|
|
|
253 This means that it is impossible to get out of sync, or more
|
|
|
254 specifically:
|
|
|
255
|
|
|
256 1. Given any byte position, the beginning of the character it is
|
|
|
257 within can be determined in constant time.
|
|
|
258 2. Given any byte position at the beginning of a character, the
|
|
|
259 beginning of the next character can be determined in constant
|
|
|
260 time.
|
|
|
261 3. Given any byte position at the beginning of a character, the
|
|
|
262 beginning of the previous character can be determined in constant
|
|
|
263 time.
|
|
|
264 4. Textual searches can simply treat encoded strings as if they
|
|
|
265 were encoded in a one-byte-per-character fashion rather than
|
|
|
266 the actual multi-byte encoding.
|
|
|
267
|
|
|
268 None of the standard non-modal encodings meet all of these
|
|
|
269 conditions. For example, EUC satisfies only (2) and (3), while
|
|
|
270 Shift-JIS and Big5 (not yet described) satisfy only (2). (All
|
|
|
271 non-modal encodings must satisfy (2), in order to be unambiguous.)
|
|
|
272
|
|
|
273 B. Internal Character Encoding
|
|
|
274
|
|
|
275 One 19-bit word represents a single character. The word is
|
|
|
276 separated into three fields:
|
|
|
277
|
|
|
278 Bit number: 18 17 16 15 14 13 12 11 10 09 08 07 06 05 04 03 02 01 00
|
|
|
279 <------------> <------------------> <------------------>
|
|
|
280 Field: 1 2 3
|
|
|
281
|
|
|
282 Note that fields 2 and 3 hold 7 bits each, while field 1 holds 5 bits.
|
|
|
283
|
|
|
284 Character set Field 1 Field 2 Field 3
|
|
|
285 ------------- ------- ------- -------
|
|
|
286 ASCII 0 0 PC1
|
|
|
287 range: (00 - 7F)
|
|
|
288 Control-1 0 1 PC1
|
|
|
289 range: (00 - 1F)
|
|
|
290 Dimension-1 official 0 LB - 0x80 PC1
|
|
|
291 range: (01 - 0D) (20 - 7F)
|
|
|
292 Dimension-1 private 0 LB - 0x80 PC1
|
|
|
293 range: (20 - 6F) (20 - 7F)
|
|
|
294 Dimension-2 official LB - 0x8F PC1 PC2
|
|
|
295 range: (01 - 0A) (20 - 7F) (20 - 7F)
|
|
|
296 Dimension-2 private LB - 0xE1 PC1 PC2
|
|
|
297 range: (0F - 1E) (20 - 7F) (20 - 7F)
|
|
|
298 Composite 0x1F ? ?
|
|
|
299
|
|
|
300 Note that character codes 0 - 255 are the same as the "binary encoding"
|
|
|
301 described above.
|
|
|
302 */
|
|
|
303
|
|
|
304 /*
|
|
|
305 About Unicode support:
|
|
|
306
|
|
|
307 Adding Unicode support is very desirable. Unicode will likely be a
|
|
|
308 very common representation in the future, and thus we should
|
|
|
309 represent Unicode characters using three bytes instead of four.
|
|
|
310 This means we need to find leading bytes for Unicode. Given that
|
|
|
311 there are 65,536 characters in Unicode and we can attach 96x96 =
|
|
|
312 9,216 characters per leading byte, we need eight leading bytes for
|
|
|
313 Unicode. We currently have four free (0x9A - 0x9D), and with a
|
|
|
314 little bit of rearranging we can get five: ASCII doesn't really
|
|
|
315 need to take up a leading byte. (We could just as well use 0x7F,
|
|
|
316 with a little change to the functions that assume that 0x80 is the
|
|
|
317 lowest leading byte.) This means we still need to dump three
|
|
|
318 leading bytes and move them into private space. The CNS charsets
|
|
|
319 are good candidates since they are rarely used, and
|
|
|
320 JAPANESE_JISX0208_1978 is becoming less and less used and could
|
|
|
321 also be dumped. */
|
|
|
322
|
|
|
323
|
|
|
324 /************************************************************************/
|
|
|
325 /* Definition of leading bytes */
|
|
|
326 /************************************************************************/
|
|
|
327
|
|
|
328 #define MIN_LEADING_BYTE 0x80
|
|
|
329 /* These need special treatment in a string and/or character */
|
|
|
330 #define LEADING_BYTE_ASCII 0x8E /* Omitted in a buffer */
|
|
|
331 #define LEADING_BYTE_COMPOSITE 0x80 /* for a composite character */
|
|
|
332 #define LEADING_BYTE_CONTROL_1 0x8F /* represent normal 80-9F */
|
|
|
333
|
|
|
334 /** The following are for 1-byte characters in an official charset. **/
|
|
|
335
|
|
|
336 #define LEADING_BYTE_LATIN_1 0x81 /* Right half of ISO 8859-1 */
|
|
|
337 #define LEADING_BYTE_LATIN_2 0x82 /* Right half of ISO 8859-2 */
|
|
|
338 #define LEADING_BYTE_LATIN_3 0x83 /* Right half of ISO 8859-3 */
|
|
|
339 #define LEADING_BYTE_LATIN_4 0x84 /* Right half of ISO 8859-4 */
|
|
|
340 #define LEADING_BYTE_THAI 0x85 /* TIS620-2533 */
|
|
|
341 #define LEADING_BYTE_GREEK 0x86 /* Right half of ISO 8859-7 */
|
|
|
342 #define LEADING_BYTE_ARABIC 0x87 /* Right half of ISO 8859-6 */
|
|
|
343 #define LEADING_BYTE_HEBREW 0x88 /* Right half of ISO 8859-8 */
|
|
|
344 #define LEADING_BYTE_JAPANESE_JISX0201_KANA 0x89 /* Right half of JIS X0201-1976 */
|
|
|
345 #define LEADING_BYTE_JAPANESE_JISX0201_ROMAN 0x8A /* Left half of JIS X0201-1976 */
|
|
|
346 #define LEADING_BYTE_CYRILLIC 0x8C /* Right half of ISO 8859-5 */
|
|
|
347 #define LEADING_BYTE_LATIN_5 0x8D /* Right half of ISO 8859-9 */
|
|
|
348
|
|
|
349 #define MIN_LEADING_BYTE_OFFICIAL_1 LEADING_BYTE_LATIN_1
|
|
|
350 #define MAX_LEADING_BYTE_OFFICIAL_1 LEADING_BYTE_LATIN_5
|
|
|
351
|
|
|
352 /** The following are for 2-byte characters in an official charset. **/
|
|
|
353
|
|
|
354 #define LEADING_BYTE_JAPANESE_JISX0208_1978 0x90/* Japanese JIS X0208-1978 */
|
|
|
355 #define LEADING_BYTE_CHINESE_GB 0x91 /* Chinese Hanzi GB2312-1980 */
|
|
|
356 #define LEADING_BYTE_JAPANESE_JISX0208 0x92 /* Japanese JIS X0208-1983 */
|
|
|
357 #define LEADING_BYTE_KOREAN_KSC5601 0x93 /* Hangul KS C5601-1987 */
|
|
|
358 #define LEADING_BYTE_JAPANESE_JISX0212 0x94 /* Japanese JIS X0212-1990 */
|
|
|
359 #define LEADING_BYTE_CHINESE_CNS11643_1 0x95 /* Chinese CNS11643 Set 1 */
|
|
|
360 #define LEADING_BYTE_CHINESE_CNS11643_2 0x96 /* Chinese CNS11643 Set 2 */
|
|
|
361 #define LEADING_BYTE_CHINESE_BIG5_1 0x97 /* Big5 Level 1 */
|
|
|
362 #define LEADING_BYTE_CHINESE_BIG5_2 0x98 /* Big5 Level 2 */
|
|
|
363 /* 0x99 unused */
|
|
|
364 /* 0x9A unused */
|
|
|
365 /* 0x9B unused */
|
|
|
366 /* 0x9C unused */
|
|
|
367 /* 0x9D unused */
|
|
|
368
|
|
|
369 #define MIN_LEADING_BYTE_OFFICIAL_2 LEADING_BYTE_JAPANESE_JISX0208_1978
|
|
|
370 #define MAX_LEADING_BYTE_OFFICIAL_2 LEADING_BYTE_CHINESE_BIG5_2
|
|
|
371
|
|
|
372 /** The following are for 1- and 2-byte characters in a private charset. **/
|
|
|
373
|
|
|
374 #define PRE_LEADING_BYTE_PRIVATE_1 0x9E /* 1-byte char-set */
|
|
|
375 #define PRE_LEADING_BYTE_PRIVATE_2 0x9F /* 2-byte char-set */
|
|
|
376
|
|
|
377 #define MIN_LEADING_BYTE_PRIVATE_1 0xA0
|
|
|
378 #define MAX_LEADING_BYTE_PRIVATE_1 0xEF
|
|
|
379 #define MIN_LEADING_BYTE_PRIVATE_2 0xF0
|
|
|
380 #define MAX_LEADING_BYTE_PRIVATE_2 0xFF
|
|
|
381
|
|
|
382 #define NUM_LEADING_BYTES 128
|
|
|
383
|
|
|
384
|
|
|
385 /************************************************************************/
|
|
|
386 /* Operations on leading bytes */
|
|
|
387 /************************************************************************/
|
|
|
388
|
|
|
389 /* Is this leading byte for a private charset? */
|
|
|
390
|
|
|
391 #define LEADING_BYTE_PRIVATE_P(lb) ((lb) >= MIN_LEADING_BYTE_PRIVATE_1)
|
|
|
392
|
|
|
393 /* Is this a prefix for a private leading byte? */
|
|
|
394
|
|
|
395 INLINE int LEADING_BYTE_PREFIX_P (unsigned char lb);
|
|
|
396 INLINE int
|
|
|
397 LEADING_BYTE_PREFIX_P (unsigned char lb)
|
|
|
398 {
|
|
|
399 return (lb == PRE_LEADING_BYTE_PRIVATE_1 ||
|
|
|
400 lb == PRE_LEADING_BYTE_PRIVATE_2);
|
|
|
401 }
|
|
|
402
|
|
|
403 /* Given a private leading byte, return the leading byte prefix stored
|
|
|
404 in a string */
|
|
|
405
|
|
|
406 #define PRIVATE_LEADING_BYTE_PREFIX(lb) \
|
|
|
407 ((lb) < MIN_LEADING_BYTE_PRIVATE_2 ? PRE_LEADING_BYTE_PRIVATE_1 \
|
|
|
408 : PRE_LEADING_BYTE_PRIVATE_2)
|
|
|
409
|
|
|
410
|
|
|
411
|
|
|
412
|
|
|
413 /************************************************************************/
|
|
|
414 /* Operations on individual bytes */
|
|
|
415 /* of any format */
|
|
|
416 /************************************************************************/
|
|
|
417
|
|
|
418 /* Argument `c' should be (unsigned int) or (unsigned char). */
|
|
|
419 /* Note that SP and DEL are not included. */
|
|
|
420
|
|
|
421 #define BYTE_ASCII_P(c) ((c) < 0x80)
|
|
|
422 #define BYTE_C0_P(c) ((c) < 0x20)
|
|
|
423 /* Do some forced casting just to make *sure* things are gotten right. */
|
|
|
424 #define BYTE_C1_P(c) ((unsigned int) ((unsigned int) (c) - 0x80) < 0x20)
|
|
|
425
|
|
|
426
|
|
|
427 /************************************************************************/
|
|
|
428 /* Operations on individual bytes */
|
|
|
429 /* in a Mule-formatted string */
|
|
|
430 /************************************************************************/
|
|
|
431
|
|
|
432 /* Does this byte represent the first byte of a character? */
|
|
|
433
|
|
|
434 #define BUFBYTE_FIRST_BYTE_P(c) ((c) < 0xA0)
|
|
|
435
|
|
|
436 /* Does this byte represent the first byte of a multi-byte character? */
|
|
|
437
|
|
|
438 #define BUFBYTE_LEADING_BYTE_P(c) BYTE_C1_P (c)
|
|
|
439
|
|
|
440
|
|
|
441 /************************************************************************/
|
|
|
442 /* Information about a particular character set */
|
|
|
443 /************************************************************************/
|
|
|
444
|
|
|
445 struct Lisp_Charset
|
|
|
446 {
|
|
|
447 struct lcrecord_header header;
|
|
|
448
|
|
|
449 Lisp_Object name;
|
|
|
450 Lisp_Object doc_string, registry;
|
|
|
451
|
|
|
452 Lisp_Object reverse_direction_charset;
|
|
|
453
|
|
|
454 Lisp_Object ccl_program;
|
|
|
455
|
|
|
456 unsigned int leading_byte :8;
|
|
|
457
|
|
|
458 /* Number of bytes (1 - 4) required in the internal representation
|
|
|
459 for characters in this character set. This is *not* the
|
|
|
460 same as the number of bytes used in the encoding (i.e.
|
|
|
461 the "dimension" of the character set). That value can
|
|
|
462 be derived from the TYPE. */
|
|
|
463 unsigned int rep_bytes :3;
|
|
|
464
|
|
|
465 /* Number of columns a character in this charset takes up, on TTY
|
|
|
466 devices. Not used for X devices. */
|
|
|
467 unsigned int columns :2;
|
|
|
468 /* Direction of this character set */
|
|
|
469 unsigned int direction :1;
|
|
|
470
|
|
|
471 /* Type of this character set (94, 96, 94x94, 96x96) */
|
|
|
472 unsigned int type :2;
|
|
|
473
|
|
|
474 /* Which half of font to be used to display this character set */
|
|
|
475 unsigned int graphic :2;
|
|
|
476
|
|
|
477 /* Final byte of this character set in ISO2022 designating escape sequence */
|
|
|
478 Bufbyte final;
|
|
|
479 };
|
|
|
480
|
|
|
481 DECLARE_LRECORD (charset, struct Lisp_Charset);
|
|
|
482 #define XCHARSET(x) XRECORD (x, charset, struct Lisp_Charset)
|
|
|
483 #define XSETCHARSET(x, p) XSETRECORD (x, p, charset)
|
|
|
484 #define CHARSETP(x) RECORDP (x, charset)
|
|
|
485 #define GC_CHARSETP(x) GC_RECORDP (x, charset)
|
|
|
486 #define CHECK_CHARSET(x) CHECK_RECORD (x, charset)
|
|
|
487 #define CONCHECK_CHARSET(x) CONCHECK_RECORD (x, charset)
|
|
|
488
|
|
|
489 #define CHARSET_TYPE_94 0 /* This charset includes 94 characters. */
|
|
|
490 #define CHARSET_TYPE_96 1 /* This charset includes 96 characters. */
|
|
|
491 #define CHARSET_TYPE_94X94 2 /* This charset includes 94x94 characters. */
|
|
|
492 #define CHARSET_TYPE_96X96 3 /* This charset includes 96x96 characters. */
|
|
|
493
|
|
|
494 #define CHARSET_LEFT_TO_RIGHT 0
|
|
|
495 #define CHARSET_RIGHT_TO_LEFT 1
|
|
|
496
|
|
|
497 #define CHARSET_NAME(cs) ((cs)->name)
|
|
|
498 #define CHARSET_LEADING_BYTE(cs) ((cs)->leading_byte)
|
|
|
499 #define CHARSET_REP_BYTES(cs) ((cs)->rep_bytes)
|
|
|
500 #define CHARSET_COLUMNS(cs) ((cs)->columns)
|
|
|
501 #define CHARSET_GRAPHIC(cs) ((cs)->graphic)
|
|
|
502 #define CHARSET_TYPE(cs) ((cs)->type)
|
|
|
503 #define CHARSET_DIRECTION(cs) ((cs)->direction)
|
|
|
504 #define CHARSET_FINAL(cs) ((cs)->final)
|
|
|
505 #define CHARSET_DOC_STRING(cs) ((cs)->doc_string)
|
|
|
506 #define CHARSET_REGISTRY(cs) ((cs)->registry)
|
|
|
507 #define CHARSET_CCL_PROGRAM(cs) ((cs)->ccl_program)
|
|
|
508 #define CHARSET_REVERSE_DIRECTION_CHARSET(cs) ((cs)->reverse_direction_charset)
|
|
|
509
|
|
|
510 INLINE int CHARSET_DIMENSION (struct Lisp_Charset *cs);
|
|
|
511 INLINE int
|
|
|
512 CHARSET_DIMENSION (struct Lisp_Charset *cs)
|
|
|
513 {
|
|
|
514 return (CHARSET_TYPE (cs) == CHARSET_TYPE_94 ||
|
|
|
515 CHARSET_TYPE (cs) == CHARSET_TYPE_96) ? 1 : 2;
|
|
|
516 }
|
|
|
517
|
|
|
518 INLINE int CHARSET_CHARS (struct Lisp_Charset *cs);
|
|
|
519 INLINE int
|
|
|
520 CHARSET_CHARS (struct Lisp_Charset *cs)
|
|
|
521 {
|
|
|
522 return (CHARSET_TYPE (cs) == CHARSET_TYPE_94 ||
|
|
|
523 CHARSET_TYPE (cs) == CHARSET_TYPE_94X94) ? 94 : 96;
|
|
|
524 }
|
|
|
525
|
|
|
526 #define CHARSET_PRIVATE_P(cs) \
|
|
|
527 LEADING_BYTE_PRIVATE_P (CHARSET_LEADING_BYTE (cs))
|
|
|
528
|
|
|
529 #define XCHARSET_NAME(cs) CHARSET_NAME (XCHARSET (cs))
|
|
|
530 #define XCHARSET_REP_BYTES(cs) CHARSET_REP_BYTES (XCHARSET (cs))
|
|
|
531 #define XCHARSET_COLUMNS(cs) CHARSET_COLUMNS (XCHARSET (cs))
|
|
|
532 #define XCHARSET_GRAPHIC(cs) CHARSET_GRAPHIC (XCHARSET (cs))
|
|
|
533 #define XCHARSET_TYPE(cs) CHARSET_TYPE (XCHARSET (cs))
|
|
|
534 #define XCHARSET_DIRECTION(cs) CHARSET_DIRECTION (XCHARSET (cs))
|
|
|
535 #define XCHARSET_FINAL(cs) CHARSET_FINAL (XCHARSET (cs))
|
|
|
536 #define XCHARSET_DOC_STRING(cs) CHARSET_DOC_STRING (XCHARSET (cs))
|
|
|
537 #define XCHARSET_REGISTRY(cs) CHARSET_REGISTRY (XCHARSET (cs))
|
|
|
538 #define XCHARSET_LEADING_BYTE(cs) CHARSET_LEADING_BYTE (XCHARSET (cs))
|
|
|
539 #define XCHARSET_CCL_PROGRAM(cs) CHARSET_CCL_PROGRAM (XCHARSET (cs))
|
|
|
540 #define XCHARSET_DIMENSION(cs) CHARSET_DIMENSION (XCHARSET (cs))
|
|
|
541 #define XCHARSET_CHARS(cs) CHARSET_CHARS (XCHARSET (cs))
|
|
|
542 #define XCHARSET_PRIVATE_P(cs) CHARSET_PRIVATE_P (XCHARSET (cs))
|
|
|
543 #define XCHARSET_REVERSE_DIRECTION_CHARSET(cs) \
|
|
|
544 CHARSET_REVERSE_DIRECTION_CHARSET (XCHARSET (cs))
|
|
|
545
|
|
|
546 /* Table of charsets indexed by (leading byte - 128). */
|
|
|
547 extern Lisp_Object charset_by_leading_byte[128];
|
|
|
548
|
|
|
549 /* Table of charsets indexed by type/final-byte/direction. */
|
|
|
550 extern Lisp_Object charset_by_attributes[4][128][2];
|
|
|
551
|
|
|
552 /* Table of number of bytes in the string representation of a character
|
|
|
553 indexed by the first byte of that representation.
|
|
|
554
|
|
|
555 This value can be derived other ways -- e.g. something like
|
|
|
556
|
|
|
557 (BYTE_ASCII_P (first_byte) ? 1 :
|
|
|
558 XCHARSET_REP_BYTES (CHARSET_BY_LEADING_BYTE (first_byte)))
|
|
|
559
|
|
|
560 but it's faster this way. */
|
|
|
561 extern Bytecount rep_bytes_by_first_byte[0xA0];
|
|
|
562
|
|
|
563 #ifdef ERROR_CHECK_TYPECHECK
|
|
|
564 /* int not Bufbyte even though that is the actual type of a leading byte.
|
|
|
565 This way, out-ot-range values will get caught rather than automatically
|
|
|
566 truncated. */
|
|
|
567 INLINE Lisp_Object CHARSET_BY_LEADING_BYTE (int lb);
|
|
|
568 INLINE Lisp_Object
|
|
|
569 CHARSET_BY_LEADING_BYTE (int lb)
|
|
|
570 {
|
|
|
571 assert (lb >= 0x80 && lb <= 0xFF);
|
|
|
572 return charset_by_leading_byte[lb - 128];
|
|
|
573 }
|
|
|
574
|
|
|
575 #else
|
|
|
576
|
|
|
577 #define CHARSET_BY_LEADING_BYTE(lb) (charset_by_leading_byte[(lb) - 128])
|
|
|
578
|
|
|
579 #endif
|
|
|
580
|
|
|
581 #define CHARSET_BY_ATTRIBUTES(type, final, dir) \
|
|
|
582 (charset_by_attributes[type][final][dir])
|
|
|
583
|
|
|
584 #ifdef ERROR_CHECK_TYPECHECK
|
|
|
585
|
|
|
586 /* Number of bytes in the string representation of a character */
|
|
|
587 INLINE int REP_BYTES_BY_FIRST_BYTE (int fb);
|
|
|
588 INLINE int
|
|
|
589 REP_BYTES_BY_FIRST_BYTE (int fb)
|
|
|
590 {
|
|
|
591 assert (fb >= 0 && fb < 0xA0);
|
|
|
592 return rep_bytes_by_first_byte[fb];
|
|
|
593 }
|
|
|
594
|
|
|
595 #else
|
|
|
596 #define REP_BYTES_BY_FIRST_BYTE(fb) (rep_bytes_by_first_byte[fb])
|
|
|
597 #endif
|
|
|
598
|
|
|
599 extern Lisp_Object Vcharset_ascii;
|
|
|
600 extern Lisp_Object Vcharset_control_1;
|
|
|
601 extern Lisp_Object Vcharset_latin_1;
|
|
|
602 extern Lisp_Object Vcharset_latin_2;
|
|
|
603 extern Lisp_Object Vcharset_latin_3;
|
|
|
604 extern Lisp_Object Vcharset_latin_4;
|
|
|
605 extern Lisp_Object Vcharset_latin_5;
|
|
|
606 extern Lisp_Object Vcharset_greek;
|
|
|
607 extern Lisp_Object Vcharset_thai;
|
|
|
608 extern Lisp_Object Vcharset_arabic;
|
|
|
609 extern Lisp_Object Vcharset_hebrew;
|
|
|
610 extern Lisp_Object Vcharset_cyrillic;
|
|
|
611 extern Lisp_Object Vcharset_japanese_jisx0201_kana;
|
|
|
612 extern Lisp_Object Vcharset_japanese_jisx0201_roman;
|
|
|
613 extern Lisp_Object Vcharset_japanese_jisx0208_1978;
|
|
|
614 extern Lisp_Object Vcharset_japanese_jisx0208;
|
|
|
615 extern Lisp_Object Vcharset_japanese_jisx0212;
|
|
|
616 extern Lisp_Object Vcharset_korean_ksc5601;
|
|
|
617 extern Lisp_Object Vcharset_chinese_gb;
|
|
|
618 extern Lisp_Object Vcharset_chinese_big5_1;
|
|
|
619 extern Lisp_Object Vcharset_chinese_big5_2;
|
|
|
620 extern Lisp_Object Vcharset_chinese_cns11643_1;
|
|
|
621 extern Lisp_Object Vcharset_chinese_cns11643_2;
|
|
|
622 extern Lisp_Object Vcharset_composite;
|
|
|
623
|
|
|
624
|
|
|
625 /************************************************************************/
|
|
|
626 /* Dealing with characters */
|
|
|
627 /************************************************************************/
|
|
|
628
|
|
|
629 /* Is this character represented by more than one byte in a string? */
|
|
|
630
|
|
|
631 #define CHAR_MULTIBYTE_P(c) ((c) >= 0x80)
|
|
|
632
|
|
|
633 #define CHAR_ASCII_P(c) (!CHAR_MULTIBYTE_P (c))
|
|
|
634
|
|
|
635 /* The bit fields of character are divided into 3 parts:
|
|
|
636 FIELD1(5bits):FIELD2(7bits):FIELD3(7bits) */
|
|
|
637
|
|
|
638 #define CHAR_FIELD1_MASK (0x1F << 14)
|
|
|
639 #define CHAR_FIELD2_MASK (0x7F << 7)
|
|
|
640 #define CHAR_FIELD3_MASK 0x7F
|
|
|
641
|
|
|
642 /* Macros to access each field of a character code of C. */
|
|
|
643
|
|
|
644 #define CHAR_FIELD1(c) (((c) & CHAR_FIELD1_MASK) >> 14)
|
|
|
645 #define CHAR_FIELD2(c) (((c) & CHAR_FIELD2_MASK) >> 7)
|
|
|
646 #define CHAR_FIELD3(c) ((c) & CHAR_FIELD3_MASK)
|
|
|
647
|
|
|
648 /* Field 1, if non-zero, usually holds a leading byte for a
|
|
|
649 dimension-2 charset. Field 2, if non-zero, usually holds a leading
|
|
|
650 byte for a dimension-1 charset. */
|
|
|
651
|
|
|
652 /* Converting between field values and leading bytes. */
|
|
|
653
|
|
|
654 #define FIELD2_TO_OFFICIAL_LEADING_BYTE 0x80
|
|
|
655 #define FIELD2_TO_PRIVATE_LEADING_BYTE 0x80
|
|
|
656
|
|
|
657 #define FIELD1_TO_OFFICIAL_LEADING_BYTE 0x8F
|
|
|
658 #define FIELD1_TO_PRIVATE_LEADING_BYTE 0xE1
|
|
|
659
|
|
|
660 /* Minimum and maximum allowed values for the fields. */
|
|
|
661
|
|
|
662 #define MIN_CHAR_FIELD2_OFFICIAL \
|
|
|
663 (MIN_LEADING_BYTE_OFFICIAL_1 - FIELD2_TO_OFFICIAL_LEADING_BYTE)
|
|
|
664 #define MAX_CHAR_FIELD2_OFFICIAL \
|
|
|
665 (MAX_LEADING_BYTE_OFFICIAL_1 - FIELD2_TO_OFFICIAL_LEADING_BYTE)
|
|
|
666
|
|
|
667 #define MIN_CHAR_FIELD1_OFFICIAL \
|
|
|
668 (MIN_LEADING_BYTE_OFFICIAL_2 - FIELD1_TO_OFFICIAL_LEADING_BYTE)
|
|
|
669 #define MAX_CHAR_FIELD1_OFFICIAL \
|
|
|
670 (MAX_LEADING_BYTE_OFFICIAL_2 - FIELD1_TO_OFFICIAL_LEADING_BYTE)
|
|
|
671
|
|
|
672 #define MIN_CHAR_FIELD2_PRIVATE \
|
|
|
673 (MIN_LEADING_BYTE_PRIVATE_1 - FIELD2_TO_PRIVATE_LEADING_BYTE)
|
|
|
674 #define MAX_CHAR_FIELD2_PRIVATE \
|
|
|
675 (MAX_LEADING_BYTE_PRIVATE_1 - FIELD2_TO_PRIVATE_LEADING_BYTE)
|
|
|
676
|
|
|
677 #define MIN_CHAR_FIELD1_PRIVATE \
|
|
|
678 (MIN_LEADING_BYTE_PRIVATE_2 - FIELD1_TO_PRIVATE_LEADING_BYTE)
|
|
|
679 #define MAX_CHAR_FIELD1_PRIVATE \
|
|
|
680 (MAX_LEADING_BYTE_PRIVATE_2 - FIELD1_TO_PRIVATE_LEADING_BYTE)
|
|
|
681
|
|
|
682 /* Minimum character code of each <type> character. */
|
|
|
683
|
|
|
684 #define MIN_CHAR_OFFICIAL_TYPE9N (MIN_CHAR_FIELD2_OFFICIAL << 7)
|
|
|
685 #define MIN_CHAR_PRIVATE_TYPE9N (MIN_CHAR_FIELD2_PRIVATE << 7)
|
|
|
686 #define MIN_CHAR_OFFICIAL_TYPE9NX9N (MIN_CHAR_FIELD1_OFFICIAL << 14)
|
|
|
687 #define MIN_CHAR_PRIVATE_TYPE9NX9N (MIN_CHAR_FIELD1_PRIVATE << 14)
|
|
|
688 #define MIN_CHAR_COMPOSITION (0x1F << 14)
|
|
|
689
|
|
|
690 /* Leading byte of a character.
|
|
|
691
|
|
|
692 NOTE: This takes advantage of the fact that
|
|
|
693 FIELD2_TO_OFFICIAL_LEADING_BYTE and
|
|
|
694 FIELD2_TO_PRIVATE_LEADING_BYTE are the same.
|
|
|
695 */
|
|
|
696
|
|
|
697 INLINE Bufbyte CHAR_LEADING_BYTE (Emchar c);
|
|
|
698 INLINE Bufbyte
|
|
|
699 CHAR_LEADING_BYTE (Emchar c)
|
|
|
700 {
|
|
|
701 if (CHAR_ASCII_P (c))
|
|
|
702 return LEADING_BYTE_ASCII;
|
|
|
703 else if (c < 0xA0)
|
|
|
704 return LEADING_BYTE_CONTROL_1;
|
|
|
705 else if (c < MIN_CHAR_OFFICIAL_TYPE9NX9N)
|
|
|
706 return CHAR_FIELD2 (c) + FIELD2_TO_OFFICIAL_LEADING_BYTE;
|
|
|
707 else if (c < MIN_CHAR_PRIVATE_TYPE9NX9N)
|
|
|
708 return CHAR_FIELD1 (c) + FIELD1_TO_OFFICIAL_LEADING_BYTE;
|
|
|
709 else if (c < MIN_CHAR_COMPOSITION)
|
|
|
710 return CHAR_FIELD1 (c) + FIELD1_TO_PRIVATE_LEADING_BYTE;
|
|
|
711 else
|
|
|
712 return LEADING_BYTE_COMPOSITE;
|
|
|
713 }
|
|
|
714
|
|
|
715 #define CHAR_CHARSET(c) CHARSET_BY_LEADING_BYTE (CHAR_LEADING_BYTE (c))
|
|
|
716
|
|
|
717 /* Return a character whose charset is CHARSET and position-codes
|
|
|
718 are C1 and C2. TYPE9N character ignores C2.
|
|
|
719
|
|
|
720 NOTE: This takes advantage of the fact that
|
|
|
721 FIELD2_TO_OFFICIAL_LEADING_BYTE and
|
|
|
722 FIELD2_TO_PRIVATE_LEADING_BYTE are the same.
|
|
|
723 */
|
|
|
724
|
|
|
725 INLINE Emchar MAKE_CHAR (Lisp_Object charset, int c1, int c2);
|
|
|
726 INLINE Emchar
|
|
|
727 MAKE_CHAR (Lisp_Object charset, int c1, int c2)
|
|
|
728 {
|
|
|
729 if (EQ (charset, Vcharset_ascii))
|
|
|
730 return c1;
|
|
|
731 else if (EQ (charset, Vcharset_control_1))
|
|
|
732 return c1 | 0x80;
|
|
|
733 else if (EQ (charset, Vcharset_composite))
|
|
|
734 return (0x1F << 14) | ((c1) << 7) | (c2);
|
|
|
735 else if (XCHARSET_DIMENSION (charset) == 1)
|
|
|
736 return ((XCHARSET_LEADING_BYTE (charset) -
|
|
|
737 FIELD2_TO_OFFICIAL_LEADING_BYTE) << 7) | (c1);
|
|
|
738 else if (!XCHARSET_PRIVATE_P (charset))
|
|
|
739 return ((XCHARSET_LEADING_BYTE (charset) -
|
|
|
740 FIELD1_TO_OFFICIAL_LEADING_BYTE) << 14) | ((c1) << 7) | (c2);
|
|
|
741 else
|
|
|
742 return ((XCHARSET_LEADING_BYTE (charset) -
|
|
|
743 FIELD1_TO_PRIVATE_LEADING_BYTE) << 14) | ((c1) << 7) | (c2);
|
|
|
744 }
|
|
|
745
|
|
|
746 /* The charset of character C is set to CHARSET, and the
|
|
|
747 position-codes of C are set to C1 and C2. C2 of TYPE9N character
|
|
|
748 is 0. */
|
|
|
749
|
|
|
750 /* BREAKUP_CHAR_1_UNSAFE assumes that the charset has already been
|
|
|
751 calculated, and just computes c1 and c2.
|
|
|
752
|
|
|
753 BREAKUP_CHAR also computes and stores the charset. */
|
|
|
754
|
|
|
755 #define BREAKUP_CHAR_1_UNSAFE(c, charset, c1, c2) \
|
|
|
756 XCHARSET_DIMENSION (charset) == 1 \
|
|
|
757 ? ((c1) = CHAR_FIELD3 (c), (c2) = 0) \
|
|
|
758 : ((c1) = CHAR_FIELD2 (c), \
|
|
|
759 (c2) = CHAR_FIELD3 (c))
|
|
|
760
|
|
|
761 INLINE void breakup_char_1 (Emchar c, Lisp_Object *charset, int *c1, int *c2);
|
|
|
762 INLINE void
|
|
|
763 breakup_char_1 (Emchar c, Lisp_Object *charset, int *c1, int *c2)
|
|
|
764 {
|
|
|
765 *charset = CHAR_CHARSET (c);
|
|
|
766 BREAKUP_CHAR_1_UNSAFE (c, *charset, *c1, *c2);
|
|
|
767 }
|
|
|
768
|
|
|
769 #define BREAKUP_CHAR(c, charset, c1, c2) \
|
|
|
770 breakup_char_1 (c, &(charset), &(c1), &(c2))
|
|
|
771
|
|
|
772
|
|
|
773
|
|
|
774 /************************************************************************/
|
|
|
775 /* Composite characters */
|
|
|
776 /************************************************************************/
|
|
|
777
|
|
|
778 extern Lisp_Object Vcomposite_char_int2string_hashtable;
|
|
|
779 extern Lisp_Object Vcomposite_char_string2int_hashtable;
|
|
|
780
|
|
|
781 Emchar lookup_composite_char (Bufbyte *str, int len);
|
|
|
782 Lisp_Object composite_char_string (Emchar ch);
|
|
|
783
|
|
|
784
|
|
|
785
|
|
|
786 /************************************************************************/
|
|
|
787 /* Exported functions */
|
|
|
788 /************************************************************************/
|
|
|
789
|
|
|
790 Lisp_Object Fget_charset (Lisp_Object);
|
|
|
791 Lisp_Object Ffind_charset (Lisp_Object);
|
|
|
792
|
|
|
793 int copy_internal_to_external (CONST Bufbyte *internal, Bytecount len,
|
|
|
794 unsigned char *external);
|
|
|
795 Bytecount copy_external_to_internal (CONST unsigned char *external,
|
|
|
796 int len, Bufbyte *internal);
|
|
|
797
|
|
|
798 #endif /* _XEMACS_MULE_CHARSET_H */
|
