comparison src/unicode.c @ 2367:ecf1ebac70d8

[xemacs-hg @ 2004-11-04 23:05:23 by ben] commit mega-patch configure.in: Turn off -Winline and -Wchar-subscripts. Use the right set of cflags when compiling modules. Rewrite ldap configuration to separate the inclusion of lber (needed in recent Cygwin) from the basic checks for the needed libraries. add a function for MAKE_JUNK_C; initially code was added to generate xemacs.def using this, but it will need to be rewritten. add an rm -f for junk.c to avoid weird Cygwin bug with cp -f onto an existing file. Sort list of auto-detected functions and eliminate unused checks for stpcpy, setlocale and getwd. Add autodetection of Cygwin scanf problems BETA: Rewrite section on configure to indicate what flags are important and what not. digest-doc.c, make-dump-id.c, profile.c, sorted-doc.c: Add proper decls for main(). make-msgfile.c: Document that this is old junk. Move proposal to text.c. make-msgfile.lex: Move proposal to text.c. make-mswin-unicode.pl: Convert error-generating code so that the entire message will be seen as a single unrecognized token. mule/mule-ccl.el: Update docs. lispref/mule.texi: Update CCL docs. ldap/eldap.c: Mule-ize. Use EXTERNAL_LIST_LOOP_2 instead of deleted EXTERNAL_LIST_LOOP. * XEmacs 21.5.18 "chestnut" is released. --------------------------------------------------------------- MULE-RELATED WORK: --------------------------------------------------------------- --------------------------- byte-char conversion --------------------------- buffer.c, buffer.h, insdel.c, text.c: Port FSF algorithm for byte-char conversion, replacing broken previous version. Track the char position of the gap. Add functions to do char-byte conversion downwards as well as upwards. Move comments about algorithm workings to internals manual. --------------------------- work on types --------------------------- alloc.c, console-x-impl.h, dump-data.c, dump-data.h, dumper.c, dialog-msw.c, dired-msw.c, doc.c, editfns.c, esd.c, event-gtk.h, event-msw.c, events.c, file-coding.c, file-coding.h, fns.c, glyphs-eimage.c, glyphs-gtk.c, glyphs-msw.c, glyphs-shared.c, glyphs-x.c, glyphs.c, glyphs.h, gui.c, hpplay.c, imgproc.c, intl-win32.c, lrecord.h, lstream.c, keymap.c, lisp.h, libsst.c, linuxplay.c, miscplay.c, miscplay.h, mule-coding.c, nas.c, nt.c, ntheap.c, ntplay.c, objects-msw.c, objects-tty.c, objects-x.c, print.c, process-nt.c, process.c, redisplay.h, select-common.h, select-gtk.c, select-x.c, sgiplay.c, sound.c, sound.h, sunplay.c, sysfile.h, sysdep.c, syswindows.h, text.c, unexnt.c, win32.c, xgccache.c: Further work on types. This creates a full set of types for all the basic semantics of `char' that I have so far identified, so that its semantics can always be identified for the purposes of proper Mule-safe code, and the raw use of `char' always avoided. (1) More type renaming, for consistency of naming. Char_ASCII -> Ascbyte UChar_ASCII -> UAscbyte Char_Binary -> CBinbyte UChar_Binary -> Binbyte SChar_Binary -> SBinbyte (2) Introduce Rawbyte, CRawbyte, Boolbyte, Chbyte, UChbyte, and Bitbyte and use them. (3) New types Itext, Wexttext and Textcount for separating out the concepts of bytes and textual units (different under UTF-16 and UTF-32, which are potential internal encodings). (4) qxestr*_c -> qxestr*_ascii. lisp.h: New; goes with other qxe() functions. #### Maybe goes in a different section. lisp.h: Group generic int-type defs together with EMACS_INT defs. lisp.h: * lisp.h (WEXTTEXT_IS_WIDE) New defns. lisp.h: New type to replace places where int occurs as a boolean. It's signed because occasionally people may want to use -1 as an error value, and because unsigned ints are viral -- see comments in the internals manual against using them. dynarr.c: int -> Bytecount. --------------------------- Mule-izing --------------------------- device-x.c: Partially Mule-ize. dumper.c, dumper.h: Mule-ize. Use Rawbyte. Use stderr_out not printf. Use wext_*(). sysdep.c, syswindows.h, text.c: New Wexttext API for manipulation of external text that may be Unicode (e.g. startup code under Windows). emacs.c: Mule-ize. Properly deal with argv in external encoding. Use wext_*() and Wexttext. Use Rawbyte. #if 0 some old junk on SCO that is unlikely to be correct. Rewrite allocation code in run-temacs. emacs.c, symsinit.h, win32.c: Rename win32 init function and call it even earlier, to initialize mswindows_9x_p even earlier, for use in startup code (XEUNICODE_P). process.c: Use _wenviron not environ under Windows, to get Unicode environment variables. event-Xt.c: Mule-ize drag-n-drop related stuff. dragdrop.c, dragdrop.h, frame-x.c: Mule-ize. text.h: Add some more stand-in defines for particular kinds of conversion; use in Mule-ization work in frame-x.c etc. --------------------------- Freshening --------------------------- intl-auto-encap-win32.c, intl-auto-encap-win32.h: Regenerate. --------------------------- Unicode-work --------------------------- intl-win32.c, syswindows.h: Factor out common options to MultiByteToWideChar and WideCharToMultiByte. Add convert_unicode_to_multibyte_malloc() and convert_unicode_to_multibyte_dynarr() and use. Add stuff for alloca() conversion of multibyte/unicode. alloc.c: Use dfc_external_data_len() in case of unicode coding system. alloc.c, mule-charset.c: Don't zero out and reinit charset Unicode tables. This fucks up dump-time loading. Anyway, either we load them at dump time or run time, never both. unicode.c: Dump the blank tables as well. --------------------------------------------------------------- DOCUMENTATION, MOSTLY MULE-RELATED: --------------------------------------------------------------- EmacsFrame.c, emodules.c, event-Xt.c, fileio.c, input-method-xlib.c, mule-wnnfns.c, redisplay-gtk.c, redisplay-tty.c, redisplay-x.c, regex.c, sysdep.c: Add comment about Mule work needed. text.h: Add more documentation describing why DFC routines were not written to return their value. Add some other DFC documentation. console-msw.c, console-msw.h: Add pointer to docs in win32.c. emacs.c: Add comments on sources of doc info. text.c, charset.h, unicode.c, intl-win32.c, intl-encap-win32.c, text.h, file-coding.c, mule-coding.c: Collect background comments and related to text matters and internationalization, and proposals for work to be done, in text.c or Internals manual, stuff related to specific textual API's in text.h, and stuff related to internal implementation of Unicode conversion in unicode.c. Put lots of pointers to the comments to make them easier to find. s/mingw32.h, s/win32-common.h, s/win32-native.h, s/windowsnt.h, win32.c: Add bunches of new documentation on the different kinds of builds and environments under Windows and how they work. Collect this info in win32.c. Add pointers to these docs in the relevant s/* files. emacs.c: Document places with long comments. Remove comment about exiting, move to internals manual, put in pointer. event-stream.c: Move docs about event queues and focus to internals manual, put in pointer. events.h: Move docs about event stream callbacks to internals manual, put in pointer. profile.c, redisplay.c, signal.c: Move documentation to the Internals manual. process-nt.c: Add pointer to comment in win32-native.el. lisp.h: Add comments about some comment conventions. lisp.h: Add comment about the second argument. device-msw.c, redisplay-msw.c: @@#### comments are out-of-date. --------------------------------------------------------------- PDUMP WORK (MOTIVATED BY UNICODE CHANGES) --------------------------------------------------------------- alloc.c, buffer.c, bytecode.c, console-impl.h, console.c, device.c, dumper.c, lrecord.h, elhash.c, emodules.h, events.c, extents.c, frame.c, glyphs.c, glyphs.h, mule-charset.c, mule-coding.c, objects.c, profile.c, rangetab.c, redisplay.c, specifier.c, specifier.h, window.c, lstream.c, file-coding.h, file-coding.c: PDUMP: Properly implement dump_add_root_block(), which never worked before, and is necessary for dumping Unicode tables. Pdump name changes for accuracy: XD_STRUCT_PTR -> XD_BLOCK_PTR. XD_STRUCT_ARRAY -> XD_BLOCK_ARRAY. XD_C_STRING -> XD_ASCII_STRING. *_structure_* -> *_block_*. lrecord.h: some comments added about dump_add_root_block() vs dump_add_root_block_ptr(). extents.c: remove incorrect comment about pdump problems with gap array. --------------------------------------------------------------- ALLOCATION --------------------------------------------------------------- abbrev.c, alloc.c, bytecode.c, casefiddle.c, device-msw.c, device-x.c, dired-msw.c, doc.c, doprnt.c, dragdrop.c, editfns.c, emodules.c, file-coding.c, fileio.c, filelock.c, fns.c, glyphs-eimage.c, glyphs-gtk.c, glyphs-msw.c, glyphs-x.c, gui-msw.c, gui-x.c, imgproc.c, intl-win32.c, lread.c, menubar-gtk.c, menubar.c, nt.c, objects-msw.c, objects-x.c, print.c, process-nt.c, process-unix.c, process.c, realpath.c, redisplay.c, search.c, select-common.c, symbols.c, sysdep.c, syswindows.h, text.c, text.h, ui-byhand.c: New macros {alloca,xnew}_{itext,{i,ext,raw,bin,asc}bytes} for more convenient allocation of these commonly requested items. Modify functions to use alloca_ibytes, alloca_array, alloca_extbytes, xnew_ibytes, etc. also XREALLOC_ARRAY, xnew. alloc.c: Rewrite the allocation functions to factor out repeated code. Add assertions for freeing dumped data. lisp.h: Moved down and consolidated with other allocation stuff. lisp.h, dynarr.c: New functions for allocation that's very efficient when mostly in LIFO order. lisp.h, text.c, text.h: Factor out some stuff for general use by alloca()-conversion funs. text.h, lisp.h: Fill out convenience routines for allocating various kinds of bytes and put them in lisp.h. Use them in place of xmalloc(), ALLOCA(). text.h: Fill out the convenience functions so the _MALLOC() kinds match the alloca() kinds. --------------------------------------------------------------- ERROR-CHECKING --------------------------------------------------------------- text.h: Create ASSERT_ASCTEXT_ASCII() and ASSERT_ASCTEXT_ASCII_LEN() from similar Eistring checkers and change the Eistring checkers to use them instead. --------------------------------------------------------------- MACROS IN LISP.H --------------------------------------------------------------- lisp.h: Redo GCPRO declarations. Create a "base" set of functions that can be used to generate any kind of gcpro sets -- regular, ngcpro, nngcpro, private ones used in GC_EXTERNAL_LIST_LOOP_2. buffer.c, callint.c, chartab.c, console-msw.c, device-x.c, dialog-msw.c, dired.c, extents.c, ui-gtk.c, rangetab.c, nt.c, mule-coding.c, minibuf.c, menubar-msw.c, menubar.c, menubar-gtk.c, lread.c, lisp.h, gutter.c, glyphs.c, glyphs-widget.c, fns.c, fileio.c, file-coding.c, specifier.c: Eliminate EXTERNAL_LIST_LOOP, which does not check for circularities. Use EXTERNAL_LIST_LOOP_2 instead or EXTERNAL_LIST_LOOP_3 or EXTERNAL_PROPERTY_LIST_LOOP_3 or GC_EXTERNAL_LIST_LOOP_2 (new macro). Removed/redid comments on EXTERNAL_LIST_LOOP. --------------------------------------------------------------- SPACING FIXES --------------------------------------------------------------- callint.c, hftctl.c, number-gmp.c, process-unix.c: Spacing fixes. --------------------------------------------------------------- FIX FOR GEOMETRY PROBLEM IN FIRST FRAME --------------------------------------------------------------- unicode.c: Add workaround for newlib bug in sscanf() [should be fixed by release 1.5.12 of Cygwin]. toolbar.c: bug fix for problem of initial frame being 77 chars wide on Windows. will be overridden by my other ws. --------------------------------------------------------------- FIX FOR LEAKING PROCESS HANDLES: --------------------------------------------------------------- process-nt.c: Fixes for leaking handles. Inspired by work done by Adrian Aichner <adrian@xemacs.org>. --------------------------------------------------------------- FIX FOR CYGWIN BUG (Unicode-related): --------------------------------------------------------------- unicode.c: Add workaround for newlib bug in sscanf() [should be fixed by release 1.5.12 of Cygwin]. --------------------------------------------------------------- WARNING FIXES: --------------------------------------------------------------- console-stream.c: `reinit' is unused. compiler.h, event-msw.c, frame-msw.c, intl-encap-win32.c, text.h: Add stuff to deal with ANSI-aliasing warnings I got. regex.c: Gather includes together to avoid warning. --------------------------------------------------------------- CHANGES TO INITIALIZATION ROUTINES: --------------------------------------------------------------- buffer.c, emacs.c, console.c, debug.c, device-x.c, device.c, dragdrop.c, emodules.c, eval.c, event-Xt.c, event-gtk.c, event-msw.c, event-stream.c, event-tty.c, events.c, extents.c, faces.c, file-coding.c, fileio.c, font-lock.c, frame-msw.c, glyphs-widget.c, glyphs.c, gui-x.c, insdel.c, lread.c, lstream.c, menubar-gtk.c, menubar-x.c, minibuf.c, mule-wnnfns.c, objects-msw.c, objects.c, print.c, scrollbar-x.c, search.c, select-x.c, text.c, undo.c, unicode.c, window.c, symsinit.h: Call reinit_*() functions directly from emacs.c, for clarity. Factor out some redundant init code. Move disallowed stuff that had crept into vars_of_glyphs() into complex_vars_of_glyphs(). Call init_eval_semi_early() from eval.c not in the middle of vars_of_() in emacs.c since there should be no order dependency in the latter calls. --------------------------------------------------------------- ARMAGEDDON: --------------------------------------------------------------- alloc.c, emacs.c, lisp.h, print.c: Rename inhibit_non_essential_printing_operations to inhibit_non_essential_conversion_operations. text.c: Assert on !inhibit_non_essential_conversion_operations. console-msw.c, print.c: Don't do conversion in SetConsoleTitle or FindWindow to avoid problems during armageddon. Put #errors for NON_ASCII_INTERNAL_FORMAT in places where problems would arise. --------------------------------------------------------------- CHANGES TO THE BUILD PROCEDURE: --------------------------------------------------------------- config.h.in, s/cxux.h, s/usg5-4-2.h, m/powerpc.h: Add comment about correct ordering of this file. Rearrange everything to follow this -- put all #undefs together and before the s&m files. Add undefs for HAVE_ALLOCA, C_ALLOCA, BROKEN_ALLOCA_IN_FUNCTION_CALLS, STACK_DIRECTION. Remove unused HAVE_STPCPY, HAVE_GETWD, HAVE_SETLOCALE. m/gec63.h: Deleted; totally broken, not used at all, not in FSF. m/7300.h, m/acorn.h, m/alliant-2800.h, m/alliant.h, m/altos.h, m/amdahl.h, m/apollo.h, m/att3b.h, m/aviion.h, m/celerity.h, m/clipper.h, m/cnvrgnt.h, m/convex.h, m/cydra5.h, m/delta.h, m/delta88k.h, m/dpx2.h, m/elxsi.h, m/ews4800r.h, m/gould.h, m/hp300bsd.h, m/hp800.h, m/hp9000s300.h, m/i860.h, m/ibmps2-aix.h, m/ibmrs6000.h, m/ibmrt-aix.h, m/ibmrt.h, m/intel386.h, m/iris4d.h, m/iris5d.h, m/iris6d.h, m/irist.h, m/isi-ov.h, m/luna88k.h, m/m68k.h, m/masscomp.h, m/mg1.h, m/mips-nec.h, m/mips-siemens.h, m/mips.h, m/news.h, m/nh3000.h, m/nh4000.h, m/ns32000.h, m/orion105.h, m/pfa50.h, m/plexus.h, m/pmax.h, m/powerpc.h, m/pyrmips.h, m/sequent-ptx.h, m/sequent.h, m/sgi-challenge.h, m/symmetry.h, m/tad68k.h, m/tahoe.h, m/targon31.h, m/tekxd88.h, m/template.h, m/tower32.h, m/tower32v3.h, m/ustation.h, m/vax.h, m/wicat.h, m/xps100.h: Delete C_ALLOCA, HAVE_ALLOCA, STACK_DIRECTION, BROKEN_ALLOCA_IN_FUNCTION_CALLS. All of this is auto-detected. When in doubt, I followed recent FSF sources, which also have these things deleted.
author ben
date Thu, 04 Nov 2004 23:08:28 +0000
parents ba4677f54a05
children 3d8143fc88e1
comparison
equal deleted inserted replaced
2366:2a392e0c390a 2367:ecf1ebac70d8
1 /* Code to handle Unicode conversion. 1 /* Code to handle Unicode conversion.
2 Copyright (C) 2000, 2001, 2002, 2003 Ben Wing. 2 Copyright (C) 2000, 2001, 2002, 2003, 2004 Ben Wing.
3 3
4 This file is part of XEmacs. 4 This file is part of XEmacs.
5 5
6 XEmacs is free software; you can redistribute it and/or modify it 6 XEmacs is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the 7 under the terms of the GNU General Public License as published by the
40 #include "charset.h" 40 #include "charset.h"
41 #include "file-coding.h" 41 #include "file-coding.h"
42 #include "opaque.h" 42 #include "opaque.h"
43 43
44 #include "sysfile.h" 44 #include "sysfile.h"
45
46 /* For more info about how Unicode works under Windows, see intl-win32.c. */
47
48 /* Info about Unicode translation tables [ben]:
49
50 FORMAT:
51 -------
52
53 We currently use the following format for tables:
54
55 If dimension == 1, to_unicode_table is a 96-element array of ints
56 (Unicode code points); else, it's a 96-element array of int * pointers,
57 each of which points to a 96-element array of ints. If no elements in a
58 row have been filled in, the pointer will point to a default empty
59 table; that way, memory usage is more reasonable but lookup still fast.
60
61 -- If from_unicode_levels == 1, from_unicode_table is a 256-element
62 array of shorts (octet 1 in high byte, octet 2 in low byte; we don't
63 store Ichars directly to save space).
64
65 -- If from_unicode_levels == 2, from_unicode_table is a 256-element
66 array of short * pointers, each of which points to a 256-element array
67 of shorts.
68
69 -- If from_unicode_levels == 3, from_unicode_table is a 256-element
70 array of short ** pointers, each of which points to a 256-element array
71 of short * pointers, each of which points to a 256-element array of
72 shorts.
73
74 -- If from_unicode_levels == 4, same thing but one level deeper.
75
76 Just as for to_unicode_table, we use default tables to fill in all
77 entries with no values in them.
78
79 #### An obvious space-saving optimization is to use variable-sized
80 tables, where each table instead of just being a 256-element array, is a
81 structure with a start value, an end value, and a variable number of
82 entries (END - START + 1). Only 8 bits are needed for END and START,
83 and could be stored at the end to avoid alignment problems. However,
84 before charging off and implementing this, we need to consider whether
85 it's worth it:
86
87 (1) Most tables will be highly localized in which code points are
88 defined, heavily reducing the possible memory waste. Before doing any
89 rewriting, write some code to see how much memory is actually being
90 wasted (i.e. ratio of empty entries to total # of entries) and only
91 start rewriting if it's unacceptably high. You have to check over all
92 charsets.
93
94 (2) Since entries are usually added one at a time, you have to be very
95 careful when creating the tables to avoid realloc()/free() thrashing in
96 the common case when you are in an area of high localization and are
97 going to end up using most entries in the table. You'd certainly want
98 to allow only certain sizes, not arbitrary ones (probably powers of 2,
99 where you want the entire block including the START/END values to fit
100 into a power of 2, minus any malloc overhead if there is any -- there's
101 none under gmalloc.c, and probably most system malloc() functions are
102 quite smart nowadays and also have no overhead). You could optimize
103 somewhat during the in-C initializations, because you can compute the
104 actual usage of various tables by scanning the entries you're going to
105 add in a separate pass before adding them. (You could actually do the
106 same thing when entries are added on the Lisp level by making the
107 assumption that all the entries will come in one after another before
108 any use is made of the data. So as they're coming in, you just store
109 them in a big long list, and the first time you need to retrieve an
110 entry, you compute the whole table at once.) You'd still have to deal
111 with the possibility of later entries coming in, though.
112
113 (3) You do lose some speed using START/END values, since you need a
114 couple of comparisons at each level. This could easily make each single
115 lookup become 3-4 times slower. The Unicode book considers this a big
116 issue, and recommends against variable-sized tables for this reason;
117 however, they almost certainly have in mind applications that primarily
118 involve conversion of large amounts of data. Most Unicode strings that
119 are translated in XEmacs are fairly small. The only place where this
120 might matter is in loading large files -- e.g. a 3-megabyte
121 Unicode-encoded file. So think about this, and maybe do a trial
122 implementation where you don't worry too much about the intricacies of
123 (2) and just implement some basic "multiply by 1.5" trick or something
124 to do the resizing. There is a very good FAQ on Unicode called
125 something like the Linux-Unicode How-To (it should be part of the Linux
126 How-To's, I think), that lists the url of a guy with a whole bunch of
127 unicode files you can use to stress-test your implementations, and he's
128 highly likely to have a good multi-megabyte Unicode-encoded file (with
129 normal text in it -- if you created your own just by creating repeated
130 strings of letters and numbers, you probably wouldn't get accurate
131 results).
132
133 INITIALIZATION:
134 ---------------
135
136 There are advantages and disadvantages to loading the tables at
137 run-time.
138
139 Advantages:
140
141 They're big, and it's very fast to recreate them (a fraction of a second
142 on modern processors).
143
144 Disadvantages:
145
146 (1) User-defined charsets: It would be inconvenient to require all
147 dumped user-defined charsets to be reloaded at init time.
148
149 (2) Starting up in a non-ISO-8859-1 directory. If we load at run-time,
150 we don't load the tables until after we've parsed the current
151 directories, and we run into a real bootstrapping problem, if the
152 directories themselves are non-ISO-8859-1. This is potentially fixable
153 once we switch to using Unicode internally, so we don't have to do any
154 conversion (other than the automatic kind, e.g. UTF-16 to UTF-8).
155
156 NB With run-time loading, we load in init-mule-at-startup, in
157 mule-cmds.el. This is called from startup.el, which is quite late in
158 the initialization process -- but data-directory isn't set until then.
159 With dump-time loading, you still can't dump in a Japanese directory
160 (again, until we move to Unicode internally), but this is not such an
161 imposition.
162
163
164 */
45 165
46 /* #### WARNING! The current sledgehammer routines have a fundamental 166 /* #### WARNING! The current sledgehammer routines have a fundamental
47 problem in that they can't handle two characters mapping to a 167 problem in that they can't handle two characters mapping to a
48 single Unicode codepoint or vice-versa in a single charset table. 168 single Unicode codepoint or vice-versa in a single charset table.
49 It's not clear there is any way to handle this and still make the 169 It's not clear there is any way to handle this and still make the
59 Answer: You will get an abort(), since the purpose of the sledgehammer 179 Answer: You will get an abort(), since the purpose of the sledgehammer
60 routines is self-checking. The above problem with non-1-to-1 mapping 180 routines is self-checking. The above problem with non-1-to-1 mapping
61 occurs in the Big5 tables, as provided by the Unicode Consortium. */ 181 occurs in the Big5 tables, as provided by the Unicode Consortium. */
62 182
63 /* #define SLEDGEHAMMER_CHECK_UNICODE */ 183 /* #define SLEDGEHAMMER_CHECK_UNICODE */
64
65 /* We currently use the following format for tables:
66
67 If dimension == 1, to_unicode_table is a 96-element array of ints
68 (Unicode code points); else, it's a 96-element array of int *
69 pointers, each of which points to a 96-element array of ints. If no
70 elements in a row have been filled in, the pointer will point to a
71 default empty table; that way, memory usage is more reasonable but
72 lookup still fast.
73
74 -- If from_unicode_levels == 1, from_unicode_table is a 256-element
75 array of shorts (octet 1 in high byte, octet 2 in low byte; we don't
76 store Ichars directly to save space).
77
78 -- If from_unicode_levels == 2, from_unicode_table is a
79 256-element array of short * pointers, each of which points to a
80 256-element array of shorts.
81
82 -- If from_unicode_levels == 3, from_unicode_table is a
83 256-element array of short ** pointers, each of which points to
84 a 256-element array of short * pointers, each of which points to
85 a 256-element array of shorts.
86
87 -- If from_unicode_levels == 4, same thing but one level deeper.
88
89 Just as for to_unicode_table, we use default tables to fill in
90 all entries with no values in them.
91
92 #### An obvious space-saving optimization is to use variable-sized
93 tables, where each table instead of just being a 256-element array,
94 is a structure with a start value, an end value, and a variable
95 number of entries (END - START + 1). Only 8 bits are needed for
96 END and START, and could be stored at the end to avoid alignment
97 problems. However, before charging off and implementing this,
98 we need to consider whether it's worth it:
99
100 (1) Most tables will be highly localized in which code points are
101 defined, heavily reducing the possible memory waste. Before
102 doing any rewriting, write some code to see how much memory is
103 actually being wasted (i.e. ratio of empty entries to total # of
104 entries) and only start rewriting if it's unacceptably high. You
105 have to check over all charsets.
106
107 (2) Since entries are usually added one at a time, you have to be
108 very careful when creating the tables to avoid realloc()/free()
109 thrashing in the common case when you are in an area of high
110 localization and are going to end up using most entries in the
111 table. You'd certainly want to allow only certain sizes, not
112 arbitrary ones (probably powers of 2, where you want the entire
113 block including the START/END values to fit into a power of 2,
114 minus any malloc overhead if there is any -- there's none under
115 gmalloc.c, and probably most system malloc() functions are quite
116 smart nowadays and also have no overhead). You could optimize
117 somewhat during the in-C initializations, because you can compute
118 the actual usage of various tables by scanning the entries you're
119 going to add in a separate pass before adding them. (You could
120 actually do the same thing when entries are added on the Lisp
121 level by making the assumption that all the entries will come in
122 one after another before any use is made of the data. So as
123 they're coming in, you just store them in a big long list, and
124 the first time you need to retrieve an entry, you compute the
125 whole table at once.) You'd still have to deal with the
126 possibility of later entries coming in, though.
127
128 (3) You do lose some speed using START/END values, since you need
129 a couple of comparisons at each level. This could easily make
130 each single lookup become 3-4 times slower. The Unicode book
131 considers this a big issue, and recommends against variable-sized
132 tables for this reason; however, they almost certainly have in
133 mind applications that primarily involve conversion of large
134 amounts of data. Most Unicode strings that are translated in
135 XEmacs are fairly small. The only place where this might matter
136 is in loading large files -- e.g. a 3-megabyte Unicode-encoded
137 file. So think about this, and maybe do a trial implementation
138 where you don't worry too much about the intricacies of (2) and
139 just implement some basic "multiply by 1.5" trick or something to
140 do the resizing. There is a very good FAQ on Unicode called
141 something like the Linux-Unicode How-To (it should be part of the
142 Linux How-To's, I think), that lists the url of a guy with a
143 whole bunch of unicode files you can use to stress-test your
144 implementations, and he's highly likely to have a good
145 multi-megabyte Unicode-encoded file (with normal text in it -- if
146 you created your own just by creating repeated strings of letters
147 and numbers, you probably wouldn't get accurate results).
148 */
149 184
150 /* When MULE is not defined, we may still need some Unicode support -- 185 /* When MULE is not defined, we may still need some Unicode support --
151 in particular, some Windows API's always want Unicode, and the way 186 in particular, some Windows API's always want Unicode, and the way
152 we've set up the Unicode encapsulation, we may as well go ahead and 187 we've set up the Unicode encapsulation, we may as well go ahead and
153 always use the Unicode versions of split API's. (It would be 188 always use the Unicode versions of split API's. (It would be
187 static const struct sized_memory_description to_unicode_level_0_desc = { 222 static const struct sized_memory_description to_unicode_level_0_desc = {
188 sizeof (int), to_unicode_level_0_desc_1 223 sizeof (int), to_unicode_level_0_desc_1
189 }; 224 };
190 225
191 static const struct memory_description to_unicode_level_1_desc_1[] = { 226 static const struct memory_description to_unicode_level_1_desc_1[] = {
192 { XD_STRUCT_PTR, 0, 96, &to_unicode_level_0_desc }, 227 { XD_BLOCK_PTR, 0, 96, &to_unicode_level_0_desc },
193 { XD_END } 228 { XD_END }
194 }; 229 };
195 230
196 static const struct sized_memory_description to_unicode_level_1_desc = { 231 static const struct sized_memory_description to_unicode_level_1_desc = {
197 sizeof (void *), to_unicode_level_1_desc_1 232 sizeof (void *), to_unicode_level_1_desc_1
198 }; 233 };
199 234
200 static const struct memory_description to_unicode_description_1[] = { 235 static const struct memory_description to_unicode_description_1[] = {
201 { XD_STRUCT_PTR, 1, 96, &to_unicode_level_0_desc }, 236 { XD_BLOCK_PTR, 1, 96, &to_unicode_level_0_desc },
202 { XD_STRUCT_PTR, 2, 96, &to_unicode_level_1_desc }, 237 { XD_BLOCK_PTR, 2, 96, &to_unicode_level_1_desc },
203 { XD_END } 238 { XD_END }
204 }; 239 };
205 240
206 /* Not static because each charset has a set of to and from tables and 241 /* Not static because each charset has a set of to and from tables and
207 needs to describe them to pdump. */ 242 needs to describe them to pdump. */
208 const struct sized_memory_description to_unicode_description = { 243 const struct sized_memory_description to_unicode_description = {
209 sizeof (void *), to_unicode_description_1 244 sizeof (void *), to_unicode_description_1
210 }; 245 };
211 246
247 /* Used only for to_unicode_blank_2 */
248 static const struct memory_description to_unicode_level_2_desc_1[] = {
249 { XD_BLOCK_PTR, 0, 96, &to_unicode_level_1_desc },
250 { XD_END }
251 };
252
212 static const struct memory_description from_unicode_level_0_desc_1[] = { 253 static const struct memory_description from_unicode_level_0_desc_1[] = {
213 { XD_END } 254 { XD_END }
214 }; 255 };
215 256
216 static const struct sized_memory_description from_unicode_level_0_desc = { 257 static const struct sized_memory_description from_unicode_level_0_desc = {
217 sizeof (short), from_unicode_level_0_desc_1 258 sizeof (short), from_unicode_level_0_desc_1
218 }; 259 };
219 260
220 static const struct memory_description from_unicode_level_1_desc_1[] = { 261 static const struct memory_description from_unicode_level_1_desc_1[] = {
221 { XD_STRUCT_PTR, 0, 256, &from_unicode_level_0_desc }, 262 { XD_BLOCK_PTR, 0, 256, &from_unicode_level_0_desc },
222 { XD_END } 263 { XD_END }
223 }; 264 };
224 265
225 static const struct sized_memory_description from_unicode_level_1_desc = { 266 static const struct sized_memory_description from_unicode_level_1_desc = {
226 sizeof (void *), from_unicode_level_1_desc_1 267 sizeof (void *), from_unicode_level_1_desc_1
227 }; 268 };
228 269
229 static const struct memory_description from_unicode_level_2_desc_1[] = { 270 static const struct memory_description from_unicode_level_2_desc_1[] = {
230 { XD_STRUCT_PTR, 0, 256, &from_unicode_level_1_desc }, 271 { XD_BLOCK_PTR, 0, 256, &from_unicode_level_1_desc },
231 { XD_END } 272 { XD_END }
232 }; 273 };
233 274
234 static const struct sized_memory_description from_unicode_level_2_desc = { 275 static const struct sized_memory_description from_unicode_level_2_desc = {
235 sizeof (void *), from_unicode_level_2_desc_1 276 sizeof (void *), from_unicode_level_2_desc_1
236 }; 277 };
237 278
238 static const struct memory_description from_unicode_level_3_desc_1[] = { 279 static const struct memory_description from_unicode_level_3_desc_1[] = {
239 { XD_STRUCT_PTR, 0, 256, &from_unicode_level_2_desc }, 280 { XD_BLOCK_PTR, 0, 256, &from_unicode_level_2_desc },
240 { XD_END } 281 { XD_END }
241 }; 282 };
242 283
243 static const struct sized_memory_description from_unicode_level_3_desc = { 284 static const struct sized_memory_description from_unicode_level_3_desc = {
244 sizeof (void *), from_unicode_level_3_desc_1 285 sizeof (void *), from_unicode_level_3_desc_1
245 }; 286 };
246 287
247 static const struct memory_description from_unicode_description_1[] = { 288 static const struct memory_description from_unicode_description_1[] = {
248 { XD_STRUCT_PTR, 1, 256, &from_unicode_level_0_desc }, 289 { XD_BLOCK_PTR, 1, 256, &from_unicode_level_0_desc },
249 { XD_STRUCT_PTR, 2, 256, &from_unicode_level_1_desc }, 290 { XD_BLOCK_PTR, 2, 256, &from_unicode_level_1_desc },
250 { XD_STRUCT_PTR, 3, 256, &from_unicode_level_2_desc }, 291 { XD_BLOCK_PTR, 3, 256, &from_unicode_level_2_desc },
251 { XD_STRUCT_PTR, 4, 256, &from_unicode_level_3_desc }, 292 { XD_BLOCK_PTR, 4, 256, &from_unicode_level_3_desc },
252 { XD_END } 293 { XD_END }
253 }; 294 };
254 295
255 /* Not static because each charset has a set of to and from tables and 296 /* Not static because each charset has a set of to and from tables and
256 needs to describe them to pdump. */ 297 needs to describe them to pdump. */
257 const struct sized_memory_description from_unicode_description = { 298 const struct sized_memory_description from_unicode_description = {
258 sizeof (void *), from_unicode_description_1 299 sizeof (void *), from_unicode_description_1
300 };
301
302 /* Used only for from_unicode_blank_4 */
303 static const struct memory_description from_unicode_level_4_desc_1[] = {
304 { XD_BLOCK_PTR, 0, 256, &from_unicode_level_3_desc },
305 { XD_END }
259 }; 306 };
260 307
261 static Lisp_Object_dynarr *unicode_precedence_dynarr; 308 static Lisp_Object_dynarr *unicode_precedence_dynarr;
262 309
263 static const struct memory_description lod_description_1[] = { 310 static const struct memory_description lod_description_1[] = {
381 memcpy (to_table, to_unicode_blank_2, 96 * sizeof (int *)); 428 memcpy (to_table, to_unicode_blank_2, 96 * sizeof (int *));
382 XCHARSET_TO_UNICODE_TABLE (charset) = to_table; 429 XCHARSET_TO_UNICODE_TABLE (charset) = to_table;
383 } 430 }
384 431
385 { 432 {
386 XCHARSET_FROM_UNICODE_TABLE (charset) = create_new_from_unicode_table (1); 433 XCHARSET_FROM_UNICODE_TABLE (charset) =
434 create_new_from_unicode_table (1);
387 XCHARSET_FROM_UNICODE_LEVELS (charset) = 1; 435 XCHARSET_FROM_UNICODE_LEVELS (charset) = 1;
388 } 436 }
389 } 437 }
390 438
391 static void 439 static void
1389 1437
1390 Also, the return value does NOT include %n storage. */ 1438 Also, the return value does NOT include %n storage. */
1391 if ((!ignore_first_column ? 1439 if ((!ignore_first_column ?
1392 sscanf (p, "%i %i%n", &cp1, &cp2, &endcount) < 2 : 1440 sscanf (p, "%i %i%n", &cp1, &cp2, &endcount) < 2 :
1393 sscanf (p, "%i %i %i%n", &dummy, &cp1, &cp2, &endcount) < 3) 1441 sscanf (p, "%i %i %i%n", &dummy, &cp1, &cp2, &endcount) < 3)
1394 || *(p + endcount + strspn (p + endcount, " \t\n\r\f"))) 1442 /* #### Temporary code! Cygwin newlib fucked up scanf() handling
1443 of numbers beginning 0x0... starting in 04/2004, in an attempt
1444 to fix another bug. A partial fix for this was put in in
1445 06/2004, but as of 10/2004 the value of ENDCOUNT returned in
1446 such case is still wrong. If this gets fixed soon, remove
1447 this code. --ben */
1448 #ifndef CYGWIN_SCANF_BUG
1449 || *(p + endcount + strspn (p + endcount, " \t\n\r\f"))
1450 #endif
1451 )
1395 { 1452 {
1396 warn_when_safe (Qunicode, Qwarning, 1453 warn_when_safe (Qunicode, Qwarning,
1397 "Unrecognized line in translation file %s:\n%s", 1454 "Unrecognized line in translation file %s:\n%s",
1398 XSTRING_DATA (filename), line); 1455 XSTRING_DATA (filename), line);
1399 continue; 1456 continue;
2403 { 2460 {
2404 REINITIALIZE_CODING_SYSTEM_TYPE (unicode); 2461 REINITIALIZE_CODING_SYSTEM_TYPE (unicode);
2405 } 2462 }
2406 2463
2407 void 2464 void
2408 reinit_vars_of_unicode (void)
2409 {
2410 #ifdef MULE
2411 init_blank_unicode_tables ();
2412 #endif /* MULE */
2413 }
2414
2415 void
2416 vars_of_unicode (void) 2465 vars_of_unicode (void)
2417 { 2466 {
2418 reinit_vars_of_unicode ();
2419
2420 Fprovide (intern ("unicode")); 2467 Fprovide (intern ("unicode"));
2421 2468
2422 #ifdef MULE 2469 #ifdef MULE
2423 staticpro (&Vlanguage_unicode_precedence_list); 2470 staticpro (&Vlanguage_unicode_precedence_list);
2424 Vlanguage_unicode_precedence_list = Qnil; 2471 Vlanguage_unicode_precedence_list = Qnil;
2425 2472
2426 staticpro (&Vdefault_unicode_precedence_list); 2473 staticpro (&Vdefault_unicode_precedence_list);
2427 Vdefault_unicode_precedence_list = Qnil; 2474 Vdefault_unicode_precedence_list = Qnil;
2428 2475
2429 unicode_precedence_dynarr = Dynarr_new (Lisp_Object); 2476 unicode_precedence_dynarr = Dynarr_new (Lisp_Object);
2430 dump_add_root_struct_ptr (&unicode_precedence_dynarr, 2477 dump_add_root_block_ptr (&unicode_precedence_dynarr,
2431 &lisp_object_dynarr_description); 2478 &lisp_object_dynarr_description);
2479
2480 init_blank_unicode_tables ();
2481
2482 /* Note that the "block" we are describing is a single pointer, and hence
2483 we could potentially use dump_add_root_block_ptr(). However, given
2484 the way the descriptions are written, we couldn't use them, and would
2485 have to write new descriptions for each of the pointers below, since
2486 we would have to make use of a description with an XD_BLOCK_ARRAY
2487 in it. */
2488
2489 dump_add_root_block (&to_unicode_blank_1, sizeof (void *),
2490 to_unicode_level_1_desc_1);
2491 dump_add_root_block (&to_unicode_blank_2, sizeof (void *),
2492 to_unicode_level_2_desc_1);
2493
2494 dump_add_root_block (&from_unicode_blank_1, sizeof (void *),
2495 from_unicode_level_1_desc_1);
2496 dump_add_root_block (&from_unicode_blank_2, sizeof (void *),
2497 from_unicode_level_2_desc_1);
2498 dump_add_root_block (&from_unicode_blank_3, sizeof (void *),
2499 from_unicode_level_3_desc_1);
2500 dump_add_root_block (&from_unicode_blank_4, sizeof (void *),
2501 from_unicode_level_4_desc_1);
2432 #endif /* MULE */ 2502 #endif /* MULE */
2433 } 2503 }