--- a/lisp/ChangeLog	Thu Mar 18 10:50:06 2010 -0500
+++ b/lisp/ChangeLog	Fri Mar 19 14:47:44 2010 -0500
@@ -1,3 +1,9 @@
+2010-03-19  Ben Wing  <ben@xemacs.org>
+
+	* diagnose.el (show-object-memory-usage-stats):
+	Rewrite to take into account non-lisp-storage statistics
+	returned by garbage-collect-1 and friends.
+
 2010-03-18  Ben Wing  <ben@xemacs.org>
 
 	* diagnose.el (show-memory-usage):

--- a/lisp/diagnose.el	Thu Mar 18 10:50:06 2010 -0500
+++ b/lisp/diagnose.el	Fri Mar 19 14:47:44 2010 -0500
@@ -186,26 +186,40 @@
   (garbage-collect)
   (let ((buffer "*object memory usage statistics*")
 	(plist (object-memory-usage-stats))
-	(fmt "%-30s%10s%10s\n")
+	(fmt "%-30s%10s%10s%18s\n")
 	(grandtotal 0)
 	begin)
   (flet ((show-stats (match-string)
-	(princ (format fmt "object" "count" "storage"))
-	(princ (make-string 50 ?-))
+	(princ (format fmt "object" "count" "storage" "non-Lisp storage"))
+	(princ (make-string 68 ?-))
 	(princ "\n")
 	(let ((total-use 0)
+	      (total-non-lisp-use 0)
 	      (total-use-overhead 0)
 	      (total-count 0))
 	  (map-plist 
 	   #'(lambda (stat num)
-	       (when (string-match match-string
-				   (symbol-name stat))
+	       (when (and (string-match match-string
+					(symbol-name stat))
+			  (let ((match (match-string
+					1 (symbol-name stat))))
+			    (or (< (length match) 9)
+				(not (equal (substring match -9)
+					    "-non-lisp")))))
 		 (let ((storage-use num)
 		       (storage-use-overhead 
 			(plist-get 
 			 plist 
 			 (intern (concat (match-string 1 (symbol-name stat))
 					 "-storage-including-overhead"))))
+		       (non-lisp-storage
+			(or (plist-get
+			     plist
+			     (intern (concat (match-string 1
+							   (symbol-name stat))
+					     "-non-lisp-storage")))
+			    0))
+
 		       (storage-count 
 			(or (loop for str in '("s-used" "es-used" "-used")
 			      for val = (plist-get
@@ -228,19 +242,21 @@
 		   (incf total-use-overhead (if storage-use-overhead 
 						storage-use-overhead 
 					      storage-use))
+		   (incf total-non-lisp-use non-lisp-storage)
 		   (incf total-count (or storage-count 0))
 		   (and (> storage-use 0)
 			(princ (format fmt
 				       (match-string 1 (symbol-name stat)) 
 				       (or storage-count "unknown")
-				       storage-use))))))
+				       storage-use
+				       non-lisp-storage))))))
 	   plist)
 	  (princ "\n")
 	  (princ (format fmt "total" 
-			 total-count total-use-overhead))
+			 total-count total-use-overhead total-non-lisp-use))
 	  (incf grandtotal total-use-overhead)
           (when-fboundp #'sort-numeric-fields
-            (sort-numeric-fields -1
+            (sort-numeric-fields -2
                                  (save-excursion
                                    (goto-char begin)
                                    (forward-line 3)
@@ -252,7 +268,7 @@
       (save-excursion
 	(set-buffer buffer)
 	(setq begin (point))
-	(princ "Allocated with lisp allocator:\n")
+	(princ "Allocated with lisp allocator or related:\n")
 	(show-stats "\\(.*\\)-storage$")
 	(princ (format "\n\ngrand total: %s\n" grandtotal)))
       grandtotal))))

--- a/src/ChangeLog	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/ChangeLog	Fri Mar 19 14:47:44 2010 -0500
@@ -1,3 +1,53 @@
+2010-03-19  Ben Wing  <ben@xemacs.org>
+
+	* alloc.c:
+	* alloc.c (struct):
+	* alloc.c (tick_lrecord_stats):
+	* alloc.c (gc_sweep_1):
+	* alloc.c (finish_object_memory_usage_stats):
+	* alloc.c (object_memory_usage_stats):
+	* alloc.c (compute_memusage_stats_length):
+	Call new memory-usage mechanism at sweep time to compute extra
+	memory utilization for all objects.  Add up the values element-by-
+	element to get an aggregrate set of statistics, where each is the
+	sum of the values of a single statistic across different objects
+	of the same type.  At end of sweep time, call
+	finish_object_memory_usage_stats() to add up all the aggreggrate
+	stats that are related to non-Lisp memory storage to compute
+	a single value, and add it to the list of values returned by
+	`garbage-collect' and `object-memory-usage-stats'.
+	
+	* buffer.c (compute_buffer_text_usage):
+	Don't crash on buffers without text (killed buffers?) and don't
+	double-count indirect buffers.
+	
+	* elhash.c:
+	* elhash.c (hash_table_objects_create):
+	* elhash.c (vars_of_elhash):
+	* symsinit.h:
+	Add memory-usage method to count the size of `hentries'.
+	
+	* emacs.c (main_1):
+	Call new functions in elhash.c, frame.c at init.
+	
+	* frame.c:
+	* frame.c (compute_frame_usage):
+	* frame.c (frame_memory_usage):
+	* frame.c (frame_objects_create):
+	* symsinit.h:
+	Add memory-usage method to count gutter display structures,
+	subwindow exposures.
+	
+	* gc.c (gc_finish):
+	* lisp.h:
+	Declare finish_object_memory_usage_stats(), call it in gc_finish().
+	
+	* lrecord.h (struct lrecord_implementation):
+	* lrecord.h (INIT_MEMORY_USAGE_STATS):
+	New value in implementation struct to track number of non-Lisp-memory
+	statistics.  Computed in alloc.c.
+	
+
 2010-03-18  Ben Wing  <ben@xemacs.org>
 
 	* alloc.c:

--- a/src/alloc.c	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/alloc.c	Fri Mar 19 14:47:44 2010 -0500
@@ -3624,6 +3624,10 @@
   int bytes_freed;
   int instances_on_free_list;
   int bytes_on_free_list;
+#ifdef MEMORY_USAGE_STATS
+  Bytecount nonlisp_bytes_in_use;
+  struct generic_usage_stats stats;
+#endif
 } lrecord_stats [countof (lrecord_implementations_table)];
 
 void
@@ -3638,6 +3642,22 @@
     case ALLOC_IN_USE:
       lrecord_stats[type_index].instances_in_use++;
       lrecord_stats[type_index].bytes_in_use += sz;
+#ifdef MEMORY_USAGE_STATS
+      {
+	struct generic_usage_stats stats;
+	Lisp_Object obj = wrap_pointer_1 (h);
+	if (HAS_OBJECT_METH_P (obj, memory_usage))
+	  {
+	    int i;
+	    int total_stats = OBJECT_PROPERTY (obj, num_extra_memusage_stats);
+	    xzero (stats);
+	    OBJECT_METH (obj, memory_usage, (obj, &stats));
+	    for (i = 0; i < total_stats; i++)
+	      lrecord_stats[type_index].stats.othervals[i] +=
+		stats.othervals[i];
+	  }
+      }
+#endif
       break;
     case ALLOC_FREE:
       lrecord_stats[type_index].instances_freed++;
@@ -4517,9 +4537,7 @@
   sweep_eval_data ();
   sweep_misc_user_data ();
 #endif /* EVENT_DATA_AS_OBJECTS */
-#endif /* not NEW_GC */
-
-#ifndef NEW_GC
+
 #ifdef PDUMP
   pdump_objects_unmark ();
 #endif
@@ -4659,6 +4677,25 @@
   strcat (buf, suffix);
 }
 
+void
+finish_object_memory_usage_stats (void)
+{
+#ifdef MEMORY_USAGE_STATS
+  int i;
+  for (i = 0; i < countof (lrecord_implementations_table); i++)
+    {
+      struct lrecord_implementation *imp = lrecord_implementations_table[i];
+      if (imp && imp->num_extra_nonlisp_memusage_stats)
+	{
+	  int j;
+	  for (j = 0; j < imp->num_extra_nonlisp_memusage_stats; j++)
+	    lrecord_stats[i].nonlisp_bytes_in_use +=
+	      lrecord_stats[i].stats.othervals[j];
+	}
+    }
+#endif /* MEMORY_USAGE_STATS */
+}
+
 static Lisp_Object
 object_memory_usage_stats (int set_total_gc_usage)
 {
@@ -4667,7 +4704,6 @@
   EMACS_INT tgu_val = 0;
 
 #ifdef NEW_GC
-  
   for (i = 0; i < countof (lrecord_implementations_table); i++)
     {
       if (lrecord_stats[i].instances_in_use != 0)
@@ -4742,6 +4778,13 @@
           sprintf (buf, "%s-storage", name);
           pl = gc_plist_hack (buf, lrecord_stats[i].bytes_in_use, pl);
 	  tgu_val += lrecord_stats[i].bytes_in_use;
+	  if (lrecord_stats[i].nonlisp_bytes_in_use)
+	    {
+	      sprintf (buf, "%s-non-lisp-storage", name);
+	      pl = gc_plist_hack (buf, lrecord_stats[i].nonlisp_bytes_in_use,
+				  pl);
+	      tgu_val += lrecord_stats[i].nonlisp_bytes_in_use;
+	    }
 	  pluralize_and_append (buf, name, "-freed");
           if (lrecord_stats[i].instances_freed != 0)
             pl = gc_plist_hack (buf, lrecord_stats[i].instances_freed, pl);
@@ -4807,6 +4850,9 @@
   for (i = 0; i < countof (lrecord_implementations_table); i++)
     {
       int len = 0;
+      int nonlisp_len = 0;
+      int seen_break = 0;
+
       struct lrecord_implementation *imp = lrecord_implementations_table[i];
 
       if (!imp)
@@ -4820,11 +4866,18 @@
 	LIST_LOOP_2 (item, imp->memusage_stats_list)
 	  {
 	    if (!NILP (item) && !EQ (item, Qt))
-	      len++;
+	      {
+		len++;
+		if (!seen_break)
+		  nonlisp_len++;
+	      }
+	    else
+	      seen_break++;
 	  }
       }
 
       imp->num_extra_memusage_stats = len;
+      imp->num_extra_nonlisp_memusage_stats = nonlisp_len;
     }
 }
 

--- a/src/buffer.c	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/buffer.c	Fri Mar 19 14:47:44 2010 -0500
@@ -1761,9 +1761,20 @@
 static Bytecount
 compute_buffer_text_usage (struct buffer *b, struct usage_stats *ustats)
 {
-  Bytecount was_requested = b->text->z - 1;
-  Bytecount gap = b->text->gap_size + b->text->end_gap_size;
-  Bytecount malloc_use = malloced_storage_size (b->text->beg, was_requested + gap, 0);
+  Bytecount was_requested, gap, malloc_use;
+
+  /* Killed buffer? */
+  if (!b->text)
+    return 0;
+
+  /* Indirect buffer shares its text with someone else, so don't double-
+     count the text */
+  if (b->base_buffer)
+    return 0;
+
+  was_requested = b->text->z - 1;
+  gap = b->text->gap_size + b->text->end_gap_size;
+  malloc_use = malloced_storage_size (b->text->beg, was_requested + gap, 0);
 
   ustats->gap_overhead    += gap;
   ustats->was_requested   += was_requested;

--- a/src/elhash.c	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/elhash.c	Fri Mar 19 14:47:44 2010 -0500
@@ -280,6 +280,28 @@
   return XHASH_TABLE (hash_table)->count;
 }
 
+#ifdef MEMORY_USAGE_STATS
+
+struct hash_table_stats
+{
+  struct usage_stats u;
+  Bytecount hentries;
+};
+
+static void
+hash_table_memory_usage (Lisp_Object hashtab,
+			 struct generic_usage_stats *gustats)
+{
+  Lisp_Hash_Table *ht = XHASH_TABLE (hashtab);
+  struct hash_table_stats *stats = (struct hash_table_stats *) gustats;
+  stats->hentries +=
+    malloced_storage_size (ht->hentries,
+			   sizeof (htentry) * (ht->size + 1),
+			   &stats->u);
+}
+
+#endif /* MEMORY_USAGE_STATS */
+
 
 /* Printing hash tables.
 
@@ -1805,6 +1827,14 @@
 /************************************************************************/
 
 void
+hash_table_objects_create (void)
+{
+#ifdef MEMORY_USAGE_STATS
+  OBJECT_HAS_METHOD (hash_table, memory_usage);
+#endif
+}
+
+void
 syms_of_elhash (void)
 {
   DEFSUBR (Fhash_table_p);
@@ -1854,6 +1884,15 @@
 }
 
 void
+vars_of_elhash (void)
+{
+#ifdef MEMORY_USAGE_STATS
+  OBJECT_HAS_PROPERTY
+    (hash_table, memusage_stats_list, list1 (intern ("hash-entries")));
+#endif /* MEMORY_USAGE_STATS */
+}
+
+void
 init_elhash_once_early (void)
 {
   INIT_LISP_OBJECT (hash_table);

--- a/src/emacs.c	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/emacs.c	Fri Mar 19 14:47:44 2010 -0500
@@ -1763,7 +1763,9 @@
       buffer_objects_create ();
       extent_objects_create ();
       face_objects_create ();
+      frame_objects_create ();
       glyph_objects_create ();
+      hash_table_objects_create ();
       lstream_objects_create ();
       mule_charset_objects_create ();
 #ifdef HAVE_GTK
@@ -2108,6 +2110,7 @@
       vars_of_dragdrop ();
 #endif
       vars_of_editfns ();
+      vars_of_elhash ();
       vars_of_emacs ();
       vars_of_eval ();
 

--- a/src/frame.c	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/frame.c	Fri Mar 19 14:47:44 2010 -0500
@@ -4064,6 +4064,53 @@
 }
 
 
+#ifdef MEMORY_USAGE_STATS
+
+struct frame_stats
+{
+  struct usage_stats u;
+  Bytecount gutter;
+  Bytecount expose_ignore;
+  Bytecount other;
+};
+
+static void
+compute_frame_usage (struct frame *f, struct frame_stats *stats,
+		     struct usage_stats *ustats)
+{
+  enum edge_pos edge;
+  EDGE_POS_LOOP (edge)
+    {
+      stats->gutter +=
+	compute_display_line_dynarr_usage (f->current_display_lines[edge],
+					   ustats);
+      stats->gutter +=
+	compute_display_line_dynarr_usage (f->desired_display_lines[edge],
+					   ustats);
+    }
+  {
+    struct expose_ignore *e;
+
+    for (e = f->subwindow_exposures; e; e = e->next)
+      stats->expose_ignore += malloced_storage_size (e, sizeof (*e), ustats);
+  }
+
+#if 0
+  stats->other += FRAMEMETH (f, frame_memory_usage, (f, ustats));
+#endif
+}
+
+static void
+frame_memory_usage (Lisp_Object frame, struct generic_usage_stats *gustats)
+{
+  struct frame_stats *stats = (struct frame_stats *) gustats;
+
+  compute_frame_usage (XFRAME (frame), stats, &stats->u);
+}
+
+#endif /* MEMORY_USAGE_STATS */
+
+
 /***************************************************************************/
 /*									   */
 /*                              initialization                             */
@@ -4071,6 +4118,14 @@
 /***************************************************************************/
 
 void
+frame_objects_create (void)
+{
+#ifdef MEMORY_USAGE_STATS
+  OBJECT_HAS_METHOD (frame, memory_usage);
+#endif
+}
+
+void
 init_frame (void)
 {
 #ifndef PDUMP
@@ -4216,6 +4271,12 @@
 void
 vars_of_frame (void)
 {
+#ifdef MEMORY_USAGE_STATS
+  OBJECT_HAS_PROPERTY
+    (frame, memusage_stats_list, list3 (Qgutter, intern ("expose-ignore"),
+					Qother));
+#endif /* MEMORY_USAGE_STATS */
+
   /* */
   Vframe_being_created = Qnil;
   staticpro (&Vframe_being_created);

--- a/src/gc.c	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/gc.c	Fri Mar 19 14:47:44 2010 -0500
@@ -1774,6 +1774,7 @@
 #ifdef NEW_GC
   GC_SET_PHASE (FINISH_GC);
 #endif /* NEW_GC */
+  finish_object_memory_usage_stats ();
   consing_since_gc = 0;
 #ifndef DEBUG_XEMACS
   /* Allow you to set it really fucking low if you really want ... */

--- a/src/lisp.h	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/lisp.h	Fri Mar 19 14:47:44 2010 -0500
@@ -4793,6 +4793,7 @@
 MODULE_API Lisp_Object listu (Lisp_Object, ...);
 DECLARE_DOESNT_RETURN (memory_full (void));
 void disksave_object_finalization (void);
+void finish_object_memory_usage_stats (void);
 extern int purify_flag;
 #ifndef NEW_GC
 extern EMACS_INT gc_generation_number[1];

--- a/src/lrecord.h	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/lrecord.h	Fri Mar 19 14:47:44 2010 -0500
@@ -544,6 +544,12 @@
      on the value placed in `memusage_stats_list'. */
   Elemcount num_extra_memusage_stats;
 
+  /* Number of additional type-specific statistics related to
+     non-Lisp-Object memory usage for this object.  Automatically
+     calculated (see compute_memusage_stats_length()) based on the value
+     placed in `memusage_stats_list'. */
+  Elemcount num_extra_nonlisp_memusage_stats;
+
   /* List of tags to be given to the extra statistics, one per statistic.
      Qnil or Qt can be present to separate off different slices.  Qnil
      separates different slices within the same type of statistics.
@@ -1420,6 +1426,8 @@
     memusage_stats_list = Qnil;						\
   lrecord_implementations_table[lrecord_type_##type]->			\
     num_extra_memusage_stats = -1;					\
+  lrecord_implementations_table[lrecord_type_##type]->			\
+    num_extra_nonlisp_memusage_stats = -1;				\
   staticpro (&lrecord_implementations_table[lrecord_type_##type]->	\
 	     memusage_stats_list);					\
 } while (0)

--- a/src/symsinit.h	Thu Mar 18 10:50:06 2010 -0500
+++ b/src/symsinit.h	Fri Mar 19 14:47:44 2010 -0500
@@ -210,7 +210,9 @@
 void buffer_objects_create (void);
 void extent_objects_create (void);
 void face_objects_create (void);
+void frame_objects_create (void);
 void glyph_objects_create (void);
+void hash_table_objects_create (void);
 void lstream_objects_create (void);
 void mule_charset_objects_create (void);
 void ui_gtk_objects_create (void);
@@ -381,6 +383,7 @@
 void vars_of_dragdrop (void);
 void vars_of_editfns (void);
 EXTERN_C void vars_of_eldap (void);
+void vars_of_elhash (void);
 void vars_of_emacs (void);
 void vars_of_eval (void);
 void reinit_vars_of_eval (void);