Mercurial > hg > xemacs-beta

changeset 973:ea6a06f7bf2c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[xemacs-hg @ 2002-08-22 14:56:23 by stephent] implement test skipping <87d6sblzat.fsf@tleepslib.sk.tsukuba.ac.jp>
author stephent
date Thu, 22 Aug 2002 14:56:32 +0000
parents 3fd7fb7868b3
children fce478afd5b4
files man/ChangeLog man/internals/internals.texi tests/ChangeLog tests/automated/syntax-tests.el tests/automated/test-harness.el
diffstat 5 files changed, 132 insertions(+), 37 deletions(-) [+]
line wrap: on
line diff
--- a/man/ChangeLog	Thu Aug 22 11:37:57 2002 +0000
+++ b/man/ChangeLog	Thu Aug 22 14:56:32 2002 +0000
@@ -1,3 +1,8 @@
+2002-08-22  Stephen J. Turnbull  <stephen@xemacs.org>
+
+	* internals/internals.texi (Regression Testing XEmacs): Document
+	how to skip and warn about tests that depend on packages.
+
 2002-08-16  Stephen J. Turnbull  <stephen@xemacs.org>
  
 	* internals/internals.texi (Regression Testing XEmacs): Fix typo.
--- a/man/internals/internals.texi	Thu Aug 22 11:37:57 2002 +0000
+++ b/man/internals/internals.texi	Thu Aug 22 14:56:32 2002 +0000
@@ -3503,6 +3503,48 @@
 @code{Check-Error}, @code{Check-Error-Message}, and @code{Check-Message}
 macros.
 
+In general, you should avoid using functionality from packages in your
+tests, because you can't be sure that everyone will have the required
+package.  However, if you've got a test that works, by all means add it.
+Simply wrap the test in an appropriate test, add a notice that the test
+was skipped, and update the @code{skipped-test-reasons} hashtable.
+Here's an example from @file{syntax-tests.el}:
+
+@example
+;; Test forward-comment at buffer boundaries
+(with-temp-buffer
+
+  ;; try to use exactly what you need: featurep, boundp, fboundp
+  (if (not (fboundp 'c-mode))
+
+      ;; We should provide a standard function for this boilerplate,
+      ;; probably called `Skip-Test' -- check for that API with C-h f
+      (let* ((reason "c-mode unavailable")
+	     (count (gethash reason skipped-test-reasons)))
+	(puthash reason (if (null count) 1 (1+ count))
+		 skipped-test-reasons)
+	(Print-Skip "comment and parse-partial-sexp tests" reason))
+
+    ;; and here's the test code
+    (c-mode)
+    (insert "// comment\n")
+    (forward-comment -2)
+    (Assert (eq (point) (point-min)))
+    (let ((point (point)))
+      (insert "/* comment */")
+      (goto-char point)
+      (forward-comment 2)
+      (Assert (eq (point) (point-max)))
+      (parse-partial-sexp point (point-max)))))
+@end example
+
+@code{Skip-Test} is intended for use with features that are normally
+present in typical configurations.  For truly optional features, or
+tests that apply to one of several alternative implementations (eg, to
+GTK widgets, but not Athena, Motif, MS Windows, or Carbon), simply
+silently omit the test.
+
+
 @node CVS Techniques, A Summary of the Various XEmacs Modules, Regression Testing XEmacs, Top
 @chapter CVS Techniques
 @cindex CVS techniques
--- a/tests/ChangeLog	Thu Aug 22 11:37:57 2002 +0000
+++ b/tests/ChangeLog	Thu Aug 22 14:56:32 2002 +0000
@@ -1,3 +1,12 @@
+2002-08-22  Stephen J. Turnbull  <stephen@xemacs.org>
+
+	* automated/test-harness.el (test-harness-from-buffer):
+	Print-Skip: new local function.
+
+	* automated/syntax-tests.el (forward-comment at buffer boundaries):
+	* automated/test-harness.el (Check-Message):
+	Warn, not barf, if required packages are unavailable.
+
 2002-07-27  Steve Youngs  <youngs@xemacs.org>
 
 	* XEmacs 21.5.8 "broccoli" is released.
--- a/tests/automated/syntax-tests.el	Thu Aug 22 11:37:57 2002 +0000
+++ b/tests/automated/syntax-tests.el	Thu Aug 22 14:56:32 2002 +0000
@@ -127,16 +127,25 @@
 
 ;; Test forward-comment at buffer boundaries
 (with-temp-buffer
-  (c-mode)
-  (insert "// comment\n")
-  (forward-comment -2)
-  (Assert (eq (point) (point-min)))
+  (if (not (fboundp 'c-mode))
+      ;; #### This whole thing should go inside a macro Skip-Test
+      (let* ((reason "c-mode unavailable")
+	     (count (gethash reason skipped-test-reasons)))
+	;;(message "%S: %S" reason count)
+	(puthash reason (if (null count) 1 (1+ count))
+		 skipped-test-reasons)
+	(Print-Skip "comment and parse-partial-sexp tests" reason))
+    (c-mode)
+    
+    (insert "// comment\n")
+    (forward-comment -2)
+    (Assert (eq (point) (point-min)))
 
-  (let ((point (point)))
-	(insert "/* comment */")
-	(goto-char point)
-	(forward-comment 2)
-	(Assert (eq (point) (point-max)))
+    (let ((point (point)))
+      (insert "/* comment */")
+      (goto-char point)
+      (forward-comment 2)
+      (Assert (eq (point) (point-max)))
 
-	;; this last used to crash
-	(parse-partial-sexp point (point-max))))
+      ;; this last used to crash
+      (parse-partial-sexp point (point-max)))))
--- a/tests/automated/test-harness.el	Thu Aug 22 11:37:57 2002 +0000
+++ b/tests/automated/test-harness.el	Thu Aug 22 14:56:32 2002 +0000
@@ -124,6 +124,11 @@
 	(missing-message-failures 0)
 	(other-failures 0)
 
+	;; #### perhaps this should be a defvar, and output at the very end
+	;; OTOH, this way AC types can use a null EMACSPACKAGEPATH to find
+	;; what stuff is needed, and ways to avoid using them
+	(skipped-test-reasons (make-hash-table :test 'equal))
+
 	(trick-optimizer nil)
 	(unexpected-test-suite-failure nil)
 	(debug-on-error t)
@@ -141,6 +146,10 @@
 	(and test-harness-verbose
 	     (princ (concat (apply #'format fmt args) "\n"))))
 
+      (defun Print-Skip (test reason &optional fmt &rest args)
+	(setq fmt (concat "SKIP: %S.  REASON: %S" fmt))
+	(princ (concat (apply #'format fmt test reason args) "\n")))
+
 
       (defmacro Assert (assertion)
 	`(condition-case error-info
@@ -201,31 +210,40 @@
 
 
       (defmacro Check-Message (expected-message-regexp &rest body)
-	(let ((quoted-body (if (= 1 (length body))
-			       `(quote ,(car body)) `(quote (progn ,@body)))))
-	  `(let ((messages ""))
-	     (defadvice message (around collect activate)
-	       (defvar messages)
-	       (let ((msg-string (apply 'format (ad-get-args 0))))
-		 (setq messages (concat messages msg-string))
-		 msg-string))
-	     (condition-case error-info
-		 (progn
-		   (setq trick-optimizer (progn ,@body))
-		   (if (string-match ,expected-message-regexp messages)
-		       (progn
-			 (Print-Pass "%S ==> value %S, message %S, matching %S, as expected"
-				     ,quoted-body trick-optimizer messages ',expected-message-regexp)
-			 (incf passes))
-		     (Print-Failure "%S ==> value %S, message %S, NOT matching expected %S"
-				    ,quoted-body  trick-optimizer messages
-				    ',expected-message-regexp)
-		     (incf missing-message-failures)))
-	       (error
-		(Print-Failure "%S ==> unexpected error %S"
-			       ,quoted-body error-info)
-		(incf other-failures)))
-	     (ad-unadvise 'message))))
+	(if (not (fboundp 'defadvice))
+	    ;; #### This whole thing should go inside a macro Skip-Test
+	    (let* ((reason "advice unavailable")
+		   (count (gethash reason skipped-test-reasons)))
+	      ;(message "%S: %S" reason count)
+	      (puthash reason (if (null count) 1 (1+ count))
+		       skipped-test-reasons)
+	      `(Print-Skip ,expected-message-regexp ,reason))
+	  (let ((quoted-body (if (= 1 (length body))
+				 `(quote ,(car body))
+			       `(quote (progn ,@body)))))
+	    `(let ((messages ""))
+	       (defadvice message (around collect activate)
+		 (defvar messages)
+		 (let ((msg-string (apply 'format (ad-get-args 0))))
+		   (setq messages (concat messages msg-string))
+		   msg-string))
+	       (condition-case error-info
+		   (progn
+		     (setq trick-optimizer (progn ,@body))
+		     (if (string-match ,expected-message-regexp messages)
+			 (progn
+			   (Print-Pass "%S ==> value %S, message %S, matching %S, as expected"
+				       ,quoted-body trick-optimizer messages ',expected-message-regexp)
+			   (incf passes))
+		       (Print-Failure "%S ==> value %S, message %S, NOT matching expected %S"
+				      ,quoted-body  trick-optimizer messages
+				      ',expected-message-regexp)
+		       (incf missing-message-failures)))
+		 (error
+		  (Print-Failure "%S ==> unexpected error %S"
+				 ,quoted-body error-info)
+		  (incf other-failures)))
+	       (ad-unadvise 'message)))))
 
       (defmacro Ignore-Ebola (&rest body)
 	`(let ((debug-issue-ebola-notices -42)) ,@body))
@@ -284,7 +302,19 @@
 	      (if (> total 0)
 		  (format "%s: %d of %d (%d%%) tests successful."
 			  basename passes total (/ (* 100 passes) total))
-		(format "%s: No tests run" basename))))
+		(format "%s: No tests run" basename)))
+	     (reasons ""))
+	(maphash (lambda (key value)
+		   (setq reasons
+			 (concat reasons
+				 (format "\n    %d tests skipped because %s"
+					 value key))))
+		 skipped-test-reasons)
+	(when (> (length reasons) 1)
+	  (setq summary-msg (concat summary-msg reasons "
+    Probably XEmacs cannot find your installed packages.  Set EMACSPACKAGEPATH
+    to the package hierarchy root or configure with --package-path to enable
+    the skipped tests.")))
 	(message "%s" summary-msg))
       (when unexpected-test-suite-failure
 	(message "Test suite execution failed unexpectedly."))