changeset 514: |
da17bf652e48 |
parent: |
4cdf71621bae
|
child: |
806c2b214df8 |
author: |
Richard Westhaver <ellis@rwest.io> |
date: |
Fri, 05 Jul 2024 22:13:39 -0400 |
permissions: |
-rw-r--r-- |
description: |
tests and light feature annotations to support darwin (no uring, no mime types) |
1 ;;; rt.lisp --- regression testing 3 ;; Regression Testing framework. inspired by PCL, the original CMUCL 4 ;; code, and the SBCL port. 8 ;; - :rt https://www.merl.com/publications/docs/TR91-04.pdf Chapter 1 9 ;; - :com.gigamonkeys.test https://github.com/gigamonkey/monkeylib-test-framework 10 ;; - :sb-rt https://github.com/sbcl/sbcl/blob/master/contrib/sb-rt/rt.lisp 12 ;; This package is intended to provide a modernized Lisp testing 13 ;; library with features found in some of the test frameworks listed 16 ;; - :it.bese.fiveam https://github.com/lispci/fiveam 17 ;; - :try https://github.com/melisgl/try 18 ;; - :rove https://github.com/fukamachi/rove 23 - [ ] benchmark support: do-bench, test-count, 30 (in-package :std-user) 42 :*default-test-suite-name* 45 ;; TODO 2023-09-04: :*test-profiler-list* not yet 52 :test-suite-designator 53 :check-suite-designator 68 :make-fixture-prototype 101 (defpackage :rt/bench 103 (:use :cl :std :log :rt) 109 (uiop:define-package :rt/cover 111 (:use :cl :std :log :rt) 113 :with-coverage :start-coverage :stop-coverage 114 :*coverage-directory* 117 (defpackage :rt/tracing 118 (:nicknames :tracing) 119 (:use :cl :std :log :rt) 126 :package-symbols-except)) 128 (defpackage :rt/flamegraph 129 (:nicknames :flamegraph) 130 (:use :cl :std :log :rt :sb-sprof) 131 (:export :save-flamegraph)) 137 (defvar *test-opts* '(optimize sb-c::instrument-consing)) 138 (defvar *compile-tests* nil 139 "When nil do not compile tests. With a value of t, tests are compiled 140 with default optimizations else the value is used to configure 141 compiler optimizations.") 142 (defvar *catch-test-errors* t "When non-nil, cause errors in a test to be caught.") 143 (defvar *test-suffix* "-TEST" "A suffix to append to every `test' defined with `deftest'.") 144 (defvar *test-suite-list* nil "List of available `test-suite' objects.") 145 (defvar *test-suite* nil "A 'test-suite-designator' which identifies the current `test-suite'.") 146 (eval-when (:compile-toplevel :load-toplevel :execute) 147 (defvar *default-test-suite-name* "default")) 148 (declaim (type (or stream boolean string) *test-input*)) 149 (defvar *test-input* nil "When non-nil, specifies an input stream or buffer for `*testing*'.") 150 (defvar *testing* nil "Testing state var.") 155 (defvar *simple-charset* "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") 157 (defun random-elt (seq) 158 (elt seq (random (length seq)))) 160 (defun random-ref (vec) 161 (aref vec (random (length vec)))) 163 (defun random-char () 164 (random-ref *simple-charset*)) 166 (defun random-chars (dim) 167 (let ((r (make-array dim :element-type 'character))) 168 (dotimes (i (array-total-size r) r) 169 (setf (row-major-aref r i) (random-char))))) 171 (defun random-byte () (random 255)) 173 (defun random-bytes (dim) 174 (let ((r (make-array dim :element-type 'octet))) 175 (dotimes (i (array-total-size r) r) 176 (setf (row-major-aref r i) (random-byte))))) 178 (eval-when (:compile-toplevel :load-toplevel :execute) 179 (defun make-test (&rest slots) 180 (apply #'make-instance 'test slots)) 181 (defun make-suite (&rest slots) 182 (apply #'make-instance 'test-suite slots))) 184 ;; TODO 2023-09-04: optimize 185 ;;(declaim (inline do-tests)) 186 (defun do-tests (&optional (suite *test-suite*) force (output *standard-output*)) 187 (if (pathnamep output) 188 (with-open-file (stream output :direction :output) 189 (do-suite (ensure-suite suite) :stream stream :force force)) 190 (do-suite (ensure-suite suite) :stream output :force force))) 192 (defvar *test-output-mutex* (sb-thread:make-mutex :name "tests-output")) 195 (defun do-tests-concurrently (&optional (suite *test-suite*) force (output *standard-output*)) 196 (declare (ignore suite force)) 197 (sb-thread:with-mutex (*test-output-mutex*) 198 (let ((stream (make-synonym-stream output))) 199 (let ((*standard-output* stream) 200 (*error-output* stream)) 203 (defun reset-tests () 206 *test-suite-list* nil 209 ;; this assumes that *test-suite* is re-initialized correctly to the 210 ;; correct test-suite object. 211 (defun continue-testing () 212 (if-let ((test *testing*)) 213 (throw '%in-test test) 214 (do-suite *test-suite*))) 216 ;; NOTE 2023-09-01: `pushnew' does not return an indication of whether 217 ;; place is changed - it returns place. This is functionally sound but 218 ;; means that if we want to do something else in the event that place 219 ;; is unchanged, we run into some friction, 220 ;; https://stackoverflow.com/questions/56228832/adapting-common-lisp-pushnew-to-return-success-failure 221 (defun spush (item lst &key (test #'equal)) 222 "Substituting `push'" 223 (declare (type function test)) 225 ((null lst) (push item lst)) 227 (if-let ((found (member item lst 233 #|(or nil '(t (cons item lst)))|#)) 235 ;; FIX 2023-08-31: spush, replace with `add-test' method. 236 ;; (declaim (inline normalize-test-name)) 237 (defun normalize-test-name (a) 238 "Return the normalized `test-suite-designator' of A." 240 (string (string-upcase a)) 241 (symbol (symbol-name a)) 242 (test-object (normalize-test-name (test-name a))) 243 (t (format nil "~A" a)))) 245 (defun test-name= (a b) 246 "Return t if A and B are similar `test-suite-designator's." 247 (let ((a (normalize-test-name a)) 248 (b (normalize-test-name b))) 251 ;; (declaim (inline assert-suite ensure-suite)) 252 (defun ensure-suite (name) 253 (if-let ((ok (member name *test-suite-list* :test #'test-name=))) 255 (when (or (eq name t) (null name)) (make-suite :name *default-test-suite-name*)))) 257 (defun check-suite-designator (suite) (check-type suite test-suite-designator)) 259 (defun assert-suite (name) 260 (check-suite-designator name) 261 (assert (ensure-suite name))) 263 (declaim (inline test-opt-key-p test-opt-valid-p)) 264 (defun test-opt-key-p (k) 265 "Test if K is a `test-opt-key'." 266 (member k '(:profile :save :stream))) 268 (defun test-opt-valid-p (f) 269 "Test if F is a valid `test-opt' form. If so, return F else nil." 270 (when (test-opt-key-p (car f)) 274 (define-condition test-failed (error) 275 ((reason :accessor fail-reason :initarg :reason :initform "unknown") 276 (name :accessor fail-name :initarg :name) 277 (form :accessor fail-form :initarg :form)) 278 (:documentation "Signaled when a test fails.") 279 (:report (lambda (c s) 280 (format s "The following expression failed: ~S~%~A." 285 (defgeneric eval-test (self) 286 (:documentation "Eval a `test'.")) 288 (defgeneric compile-test (self &key &allow-other-keys) 289 (:documentation "Compile a `test'.")) 291 (defgeneric locked-tests (self) 292 (:documentation "Return a list of locked tests in `test-suite' object SELF.")) 294 (defgeneric push-test (self place) 296 "Push `test' SELF to the value of slot ':tests' in `test-suite' object PLACE.")) 298 (defgeneric pop-test (self) 300 "Pop the first `test' from the slot-value of ':tests' in `test-suite' object SELF.")) 302 (defgeneric push-result (self place) 304 "Push object SELF to the value of slot ':results' in object PLACE.")) 306 (defgeneric pop-result (self) 308 "Pop the first `test-result' from the slot-value of ':tests' from object SELF.")) 310 (defgeneric push-fixture (self place) 312 "Push object SELF to the value of slot ':results' in object PLACE.")) 314 (defgeneric delete-test (self &key &allow-other-keys) 315 (:documentation "Delete `test' object specified by `test-object' SELF and optional keys.")) 317 (defgeneric find-test (self name &key &allow-other-keys) 318 (:documentation "Find `test' object specified by name and optional keys.")) 320 (defgeneric do-test (self &optional context) 321 (:documentation "Run test SELF, printing results to *standard-output*. The second 322 argument is an optional fixture. 324 SELF can also be a `test-suite', in which case the TESTS slot is 325 queried for the value of TEST. If TEST is not provided, pops the car 328 (defgeneric do-suite (self &key &allow-other-keys) 330 "Perform actions on `test-suite' object SELF with optional keys.")) 333 (deftype result-tag () 334 '(or (member :pass :fail :skip) null)) 336 (declaim (inline %make-test-result)) 337 (defstruct (test-result (:constructor %make-test-result) 339 (tag nil :type result-tag :read-only t) 340 (form nil :type form)) 342 (defmethod print-object ((self test-result) stream) 343 (print-unreadable-object (self stream :identity t) 344 (format stream "~A ~A" (tr-tag self) (tr-form self)))) 346 (defun make-test-result (tag &optional form) 347 (%make-test-result :tag tag :form form)) 349 (defmethod test-pass-p ((res test-result)) 350 (when (eq :pass (tr-tag res)) t)) 352 (defmethod test-fail-p ((res test-result)) 353 (when (eq :fail (tr-tag res)) t)) 355 (defmethod test-skip-p ((res test-result)) 356 (when (eq :skip (tr-tag res)) t)) 358 (defmethod print-object ((self test-result) stream) 359 (print-unreadable-object (self stream) 360 (format stream "~A ~A" 365 (defclass test-object () 366 ((name :initarg :name :initform (required-argument) :type string :accessor test-name) 367 #+nil (cached :initarg :cache :allocation :class :accessor test-cached-p :type boolean)) 368 (:documentation "Super class for all test-related objects.")) 370 (defmethod print-object ((self test-object) stream) 372 (print-unreadable-object (self stream :type t :identity t) 377 ;; HACK 2023-08-31: inherit sxp? 379 (defclass test (test-object) 380 ((fn :type symbol :accessor test-fn) 381 (bench :type (or boolean fixnum) :accessor test-bench :initform nil :initarg :bench) 382 (profile :type list :accessor test-profile :initform nil :initarg :profile) 383 (args :type list :accessor test-args :initform nil :initarg :args) 384 (declare :type list :accessor test-declare :initform nil :initarg :declare) 385 (form :initarg :form :initform nil :accessor test-form) 386 (doc :initarg :doc :type string :accessor test-doc) 387 (lock :initarg :lock :type boolean :accessor test-lock-p) 388 (persist :initarg :persist :initform nil :type boolean :accessor test-persist-p) 389 (results :initarg :results :type (array test-result) :accessor test-results)) 390 (:documentation "Test class typically made with `deftest'.")) 392 (defmethod initialize-instance ((self test) &key name) 393 ;; (debug! "building test" name) 398 (gensym *test-suffix*)))) 399 (setf (test-lock-p self) t) 400 ;; TODO 2023-09-21: we should count how many checks are in the :form 401 ;; slot and infer the array dimensions. 402 (setf (test-results self) (make-array 0 :element-type 'test-result)) 405 (defmethod print-object ((self test) stream) 406 (print-unreadable-object (self stream :type t :identity t) 407 (format stream "~A :fn ~A" 411 (defmethod push-result ((self test-result) (place test)) 412 (with-slots (results) place 413 (push self results))) 415 (defmethod pop-result ((self test)) 416 (pop (test-results self))) 418 (defmethod eval-test ((self test)) 419 (eval `(progn ,@(test-form self)))) 421 (defmethod funcall-test ((self test) &key declare) 422 (unless (functionp (test-fn self)) 423 (trace! (setf (symbol-function (test-fn self)) 425 ,(when declare `(declare ,declare)) 426 ,@(test-form self)))))) 427 (funcall (test-fn self))) 429 (defmethod compile-test ((self test) &key declare &allow-other-keys) 430 (with-compilation-unit (:policy '(optimize debug)) 434 ,(when declare `(declare ,declare)) 435 ,@(test-form self))))) 437 (defun fail! (form &optional fmt &rest args) 438 (let ((reason (and fmt (apply #'format nil fmt args)))) 439 (with-simple-restart (ignore-fail "Continue testing.") 440 (error 'test-failed :reason reason :form form)))) 442 (defmacro with-test-env (self &body body) 444 (setf (test-lock-p ,self) t) 445 (let* ((*testing* ,self) 450 (setf (test-lock-p ,self) %test-bail)) 453 (defmethod do-test ((self test) &optional fx) 454 (declare (ignorable fx)) 456 (trace! "running test: " *testing*) 458 (if-let ((opt *compile-tests*)) 459 ;; RESEARCH 2023-08-31: with-compilation-unit? 462 (setq opt *test-opts*) 463 (setq opt (push *test-opts* opt))) 464 ;; TODO 2023-09-21: handle failures here 465 (funcall (compile-test self :declare opt)) 466 (setf %test-result (make-test-result :pass (test-fn self)))) 468 (funcall-test self :declare '(optimize (debug 3) (safety 0))) 469 (setf %test-result (make-test-result :pass (test-name self))))))) 470 (if *catch-test-errors* 475 (setf %test-result (make-test-result :fail c)) 476 (return-from %test-bail %test-result)))) 480 (defmethod do-test ((self simple-string) &optional fixture) 481 (when-let ((test (find-test *test-suite* self))) 482 (do-test test fixture))) 484 (defmethod do-test ((self symbol) &optional fixture) 485 (when-let ((test (find-test *test-suite* (symbol-name self)))) 486 (do-test test fixture))) 490 ;; Our fixtures are just closures - with a pandoric environment. You 491 ;; might call it a domain-specific object protocol. 493 ;; You can build fixtures inside a test or use the push-fixture 494 ;; method on a `test-suite' object. 496 (deftype fixture () 'form) 498 (declaim (inline %make-fixture-prototype)) 499 (defstruct (fixture-prototype (:constructor %make-fixture-prototype) 501 (kind :empty :type keyword) 502 (form nil :type form)) 504 (defun make-fixture-prototype (kind form) 505 (%make-fixture-prototype :kind kind :form form)) 507 (defmacro make-fixture (letargs &body ds) 508 (let ((letargs (let-binding-transform letargs))) 512 (defmacro with-fixture ((var fx) &body body) 517 (defclass test-suite (test-object) 518 ((tests :initarg :set :initform nil :type list :accessor tests 519 :documentation "test-suite tests") 520 (results :initarg :results :initform nil :type list :accessor test-results 521 :documentation "test-suite results") 522 (stream :initarg :stream :initform *standard-output* :type stream :accessor test-stream) 523 (fixtures :initarg :fixtures :initform nil :type list :accessor test-fixtures)) 524 (:documentation "A class for collections of related `test' objects.")) 526 (defmethod print-object ((self test-suite) stream) 527 (print-unreadable-object (self stream :type t :identity t) 528 (format stream "~A [~d:~d:~d:~d]" 530 (length (tests self)) 531 (count t (map-tests self #'test-lock-p)) 532 (count t (map-tests self #'test-persist-p)) 533 (length (test-results self))))) 535 ;; (defmethod reinitialize-instance ((self test-suite) &rest initargs &key &allow-other-keys)) 537 (deftype test-suite-designator () 538 "Either nil, a symbol, a string, or a `test-suite' object." 539 '(or null symbol string test-suite keyword)) 541 (defun find-suite (name) 542 (declare (test-suite-designator name)) 543 (find name *test-suite-list* :test #'test-name=)) 545 (defmethod map-tests ((self test-suite) function) 546 ;; tests are stored in reverse order. run LIFO. 547 (mapcar function (reverse (tests self)))) 549 (defmethod push-test ((self test) (place test-suite)) 550 (push self (tests place))) 552 (defmethod pop-test ((self test-suite)) 555 (defmethod push-result ((self test-result) (place test-suite)) 556 (with-slots (results) place 557 (push self results))) 559 (defmethod pop-result ((self test-suite)) 560 (pop (test-results self))) 562 (defmethod find-test ((self test-suite) name &key (test #'test-name=)) 563 (declare (type (or string symbol) name) 564 (type function test)) 565 (find name (tests self) :test test)) 567 (defmethod do-test ((self test-suite) &optional test) 573 (string (find-test self test)) 574 (symbol (find-test self (symbol-name test))))) 575 (do-test (pop-test self))) 578 ;; HACK 2023-09-01: find better method of declaring failures from 579 ;; within the body of `deftest'. 580 (defmethod do-suite ((self test-suite) &key stream force) 581 (when stream (setf (test-stream self) stream)) 582 (with-slots (name stream) self 583 (format stream "in suite ~x:~%" 585 (format stream "; with ~A~A tests~%" 589 (count t (tests self) 590 :key (lambda (x) (or (test-lock-p x) (test-persist-p x)))))) 591 (length (tests self))) 592 ;; loop over each test, calling `do-test'. if locked or 593 ;; persistent, test is performed. if FORCE is non-nil all tests 597 (when (or force (test-lock-p x) (test-persist-p x)) 598 (let ((res (do-test x))) 599 (push-result res self) 600 (format stream "~@[~<~%~:;~:@(~S~) ~>~]~%" res))))) 601 ;; compare locked vs expected 602 (let ((locked (remove-if #'null (map-tests self (lambda (x) (when (test-lock-p x) x))))) 604 ;; collect if locked test not expected 605 (loop for r in (test-results self) 606 unless (test-pass-p r) 609 (format stream "~&No tests failed.~%") 611 ;; RESEARCH 2023-09-04: print fails ?? 612 (format stream "~&~A out of ~A ~ 613 total tests failed: ~ 617 (length (tests self)) 620 (format stream "~&~A unexpected failures: ~ 626 (finish-output stream) 627 ;; return values (PASS? LOCKED) 628 (values (not fails) locked)))) 630 (defmethod do-suite ((self string) &key stream) 631 (do-suite (ensure-suite self) :stream stream)) 633 (defmethod do-suite ((self symbol) &key stream) 634 (do-suite (ensure-suite self) :stream stream)) 636 (defmethod do-suite ((self null) &key stream) 637 (do-suite *test-suite* :stream stream)) 641 (defun %test (val &optional form) 644 (make-test-result :pass form) 645 (make-test-result :fail form)))) 646 ;; (print r *standard-output*) 649 (defmacro is (test &rest args) 652 (is (= 1 1)) ;=> #S(TEST-RESULT :TAG :PASS :FORM (= 1 1)) 653 If TEST returns a truthy value, return a PASS test-result, else return 654 a FAIL. The TEST is parameterized by ARGS which is a plist or nil. 656 If ARGS is nil, TEST is bound to to the RESULT slot of the test-result 657 and evaluated 'as-is'. 660 ARGS may contain the following keywords followed by a corresponding 669 All other values are treated as let bindings. 674 (push-result (trace! (funcall #'rt::%test ,test ',test)) *testing*) 675 (trace! (funcall #'rt::%test ,test ',test))) 676 (macrolet ((,form (test) `(let ,,(group args 2) ,test))) 677 ;; TODO 2023-09-21: does this work... 679 (push-result (trace! (funcall #'rt::%test (,form ,test) ',test) *testing*)) 680 (trace! (funcall #'rt::%test (,form ,test) ',test))))))) 682 (defmacro signals (condition-spec &body body) 683 "Generates a passing TEST-RESULT if body signals a condition of type 684 CONDITION-SPEC. BODY is evaluated in a block named NIL, CONDITION-SPEC 686 (let ((block-name (gensym))) 687 (destructuring-bind (condition &optional reason-control &rest reason-args) 688 (ensure-list condition-spec) 690 (handler-bind ((,condition (lambda (c) 692 ;; ok, body threw condition 693 ;; TODO 2023-09-05: result collectors 694 ;; (add-result 'test-passed 695 ;; :test-expr ',condition) 696 (return-from ,block-name (make-test-result :pass ',body))))) 698 (locally (declare (sb-ext:muffle-conditions warning)) 703 `(,reason-control ,@reason-args) 704 `("Failed to signal a ~S" ',condition))) 705 (return-from ,block-name nil))))) 708 (defmacro deftest (name props &body body) 709 "Build a test with NAME, parameterized by PROPS and with a test form of BODY. 711 PROPS is a plist which currently accepts the following parameters: 713 :PERSIST - re-run this test even if it passes 717 :PROFILE - enable profiling of this test 719 :SKIP - don't push this test to the current *TEST-SUITE* 721 :BENCH - enable benchmarking of this test 723 BODY is parsed with SB-INT:PARSE-BODY and will fill in documentation 724 and declarations for the test body. 726 (destructuring-bind (pr doc dec fn) 727 (multiple-value-bind (forms dec doc) 728 ;; parse body with docstring allowed 729 (parse-body (or body) :documentation t :whole t) 730 `(,props ,doc ,dec ',forms)) 731 ;; TODO 2023-09-21: parse plist 732 `(let ((obj (make-test 733 :name ,(format nil "~A" name) 735 ,@(when-let ((v (getf pr :persist))) `(:persist ,v)) 736 ,@(when-let ((v (getf pr :args))) `(:args ',v)) 737 ,@(when-let ((v (getf pr :bench))) `(:bench ,v)) 738 ,@(when-let ((v (getf pr :profile))) `(:profile ,v)) 739 ,@(when doc `(:doc ,doc)) 740 ,@(when dec `(:declare ,dec))))) 741 ,(unless (getf pr :skip) '(push-test obj *test-suite*)) 744 (defmacro defsuite (suite-name &rest props) 745 "Define a TEST-SUITE with provided keys. The object returned can be 746 enabled using the IN-SUITE macro, similiar to the DEFPACKAGE API." 747 (check-type suite-name (or symbol string)) 748 `(eval-when (:compile-toplevel :load-toplevel :execute) 749 (let ((obj (make-suite 750 :name (format nil "~A" ',suite-name) 751 ,@(when-let ((v (getf props :stream))) `(:stream ,v))))) 752 (setq *test-suite-list* (spush obj *test-suite-list* :test #'test-name=)) 755 (defmacro in-suite (name) 756 "Set *TEST-SUITE* to the TEST-SUITE object referred to by symbol 757 NAME. Return the object." 760 (setq *test-suite* (ensure-suite ,name))))