Mercurial > core / lisp/ffi/rocksdb/tests.lisp
changeset 698: |
96958d3eb5b0 |
parent: |
a304c9713a51
|
author: |
Richard Westhaver <ellis@rwest.io> |
date: |
Fri, 04 Oct 2024 22:04:59 -0400 |
permissions: |
-rw-r--r-- |
description: |
fixes |
1 ;;; rocksdb/tests.lisp --- RocksDB tests 4 (defpackage :rocksdb/tests 5 (:use :cl :std :rt :rocksdb :sb-ext :sb-alien :log)) 7 (in-package :rocksdb/tests) 16 (make-alien rocksdb-errptr)) 18 (defun rocksdb-test-dir () 19 (format nil "/tmp/~A/" (gensym "rocksdb-tests-"))) 21 (defun rocksdb-test-file () 22 (format nil "/tmp/~A" (gensym "rocksdb-test-"))) 25 (let ((default (rocksdb-options-create))) 26 (rocksdb-options-set-create-if-missing default t) 29 (defmacro with-errptr (sym &body body) 30 `(with-alien ((,sym rocksdb-errptr (make-errptr))) 33 (defmacro with-temp-db (sym (&optional (opts (test-opts)) (path (rocksdb-test-dir))) &body body) 37 (,sym (rocksdb-open opts path err))) 40 (rocksdb-close ,sym))))) 42 ;; not thread safe (gensym-counter) 43 (defun genkey (&optional prefix) (string-to-octets (symbol-name (gensym (or prefix "key"))))) 44 (defun genval (&optional prefix) (string-to-octets (symbol-name (gensym (or prefix "val"))))) 46 (defmacro with-opt ((var create destroy) &body body) 47 `(let ((,var ,create)) 48 (unwind-protect (progn ,@body) 52 (with-opt (o (rocksdb-options-create) (rocksdb-options-destroy o)) 54 (rocksdb-options-set-create-if-missing o t) 55 (rocksdb-options-get-create-if-missing o) 56 (rocksdb-options-set-create-missing-column-families o t) 57 (rocksdb-options-get-create-missing-column-families o) 58 (rocksdb-options-set-error-if-exists o t) 59 (rocksdb-options-get-error-if-exists o) 60 (rocksdb-options-set-paranoid-checks o t) 61 (rocksdb-options-get-paranoid-checks o) 62 (rocksdb-options-set-compression-options-use-zstd-dict-trainer o t) 63 (rocksdb-options-get-compression-options-use-zstd-dict-trainer o) 64 (rocksdb-options-set-enable-blob-gc o t) 65 (rocksdb-options-get-enable-blob-gc o) 66 (rocksdb-options-set-allow-ingest-behind o t) 67 (rocksdb-options-get-allow-ingest-behind o) 68 (rocksdb-options-set-skip-stats-update-on-db-open o t) 69 (rocksdb-options-get-skip-stats-update-on-db-open o) 70 (rocksdb-options-set-skip-checking-sst-file-sizes-on-db-open o t) 71 (rocksdb-options-get-skip-checking-sst-file-sizes-on-db-open o) 72 (rocksdb-options-set-enable-blob-files o t) 73 (rocksdb-options-get-enable-blob-files o) 74 (rocksdb-options-set-enable-pipelined-write o t) 75 (rocksdb-options-get-enable-pipelined-write o) 76 (rocksdb-options-set-unordered-write o t) 77 (rocksdb-options-get-unordered-write o) 78 (rocksdb-options-set-allow-mmap-reads o t) 79 (rocksdb-options-get-allow-mmap-reads o) 80 (rocksdb-options-set-allow-mmap-writes o t) 81 (rocksdb-options-get-allow-mmap-writes o) 82 (rocksdb-options-set-use-direct-reads o t) 83 (rocksdb-options-get-use-direct-reads o) 84 (rocksdb-options-set-use-direct-io-for-flush-and-compaction o t) 85 (rocksdb-options-get-use-direct-io-for-flush-and-compaction o) 86 (rocksdb-options-set-is-fd-close-on-exec o t) 87 (rocksdb-options-get-is-fd-close-on-exec o) 88 (rocksdb-options-set-inplace-update-support o t) 89 (rocksdb-options-get-inplace-update-support o) 90 (rocksdb-options-set-advise-random-on-open o t) 91 (rocksdb-options-get-advise-random-on-open o) 92 (rocksdb-options-set-atomic-flush o t) 93 (rocksdb-options-get-atomic-flush o) 94 (rocksdb-options-set-manual-wal-flush o t) 95 (rocksdb-options-get-manual-wal-flush o) 96 (rocksdb-options-set-avoid-unnecessary-blocking-io o t) 97 (rocksdb-options-get-avoid-unnecessary-blocking-io o) 98 ;; this is full-width value 0-255, not boolean 99 (rocksdb-options-set-level-compaction-dynamic-level-bytes o 20) 100 (rocksdb-options-get-level-compaction-dynamic-level-bytes o) 102 (rocksdb-options-set-compression-options-parallel-threads o 4) 103 (rocksdb-options-get-compression-options-parallel-threads o) 104 (rocksdb-options-set-info-log-level o 1) 105 (rocksdb-options-get-info-log-level o) 106 (rocksdb-options-set-max-open-files o 100) 107 (rocksdb-options-get-max-open-files o) 108 (rocksdb-options-set-max-file-opening-threads o 4) 109 (rocksdb-options-get-max-file-opening-threads o) 110 (rocksdb-options-set-compression-options-zstd-max-train-bytes o 1024) 111 (rocksdb-options-get-compression-options-zstd-max-train-bytes o) 112 (rocksdb-options-set-num-levels o 4) 113 (rocksdb-options-get-num-levels o) 114 (rocksdb-options-set-level0-file-num-compaction-trigger o 16) 115 (rocksdb-options-get-level0-file-num-compaction-trigger o) 116 (rocksdb-options-set-level0-slowdown-writes-trigger o 1024) 117 (rocksdb-options-get-level0-slowdown-writes-trigger o) 118 (rocksdb-options-set-level0-stop-writes-trigger o 1024) 119 (rocksdb-options-get-level0-stop-writes-trigger o) 120 (rocksdb-options-set-target-file-size-multiplier o 4) 121 (rocksdb-options-get-target-file-size-multiplier o) 123 (rocksdb-options-set-write-buffer-size o 1024) 124 (rocksdb-options-get-write-buffer-size o) 125 (rocksdb-options-set-db-write-buffer-size o 1024) 126 (rocksdb-options-get-db-write-buffer-size o) 128 (rocksdb-options-set-compression-options-max-dict-buffer-bytes o 1024) 129 (rocksdb-options-get-compression-options-max-dict-buffer-bytes o) 130 (rocksdb-options-set-max-total-wal-size o 1024) 131 (rocksdb-options-get-max-total-wal-size o) 132 (rocksdb-options-set-target-file-size-base o 1024) 133 (rocksdb-options-get-target-file-size-base o) 134 (rocksdb-options-set-max-bytes-for-level-base o 1024) 135 (rocksdb-options-get-max-bytes-for-level-base o) 137 ;; (rocksdb-options-set-max-bytes-for-level-multiplier o (the double-float (/ 1 3))) 139 (let ((opts (rocksdb-options-create)) 140 (wopts (rocksdb-writeoptions-create)) 141 (ropts (rocksdb-readoptions-create)) 142 (bopts (rocksdb-block-based-options-create))) 143 (rocksdb-options-set-create-if-missing opts t) 145 (rocksdb-options-destroy opts) 146 (rocksdb-writeoptions-destroy wopts) 147 (rocksdb-readoptions-destroy ropts) 148 (rocksdb-block-based-options-destroy bopts))) 151 "Test basic RocksDB functionality. Inserts KV pair into a temporary 152 DB where K and V are both Lisp strings." 153 (let* ((opts (test-opts)) 154 (path (rocksdb-test-dir)) 155 (db (rocksdb-open opts path nil)) 160 (wopts (rocksdb-writeoptions-create)) 161 (ropts (rocksdb-readoptions-create))) 162 (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen)) 163 (v (* unsigned-char) (make-alien unsigned-char vlen)) 164 (errptr rocksdb-errptr nil)) 177 (is (null-alien errptr)) 178 ;; get V from DB given K 179 (rocksdb:rocksdb-cancel-all-background-work db t) 180 (rocksdb-get db ropts k klen (make-alien size-t vlen) errptr) 181 (is (null-alien errptr)) 182 ;; copy V to RVAL and validate 183 (let ((rval (make-array vlen :element-type 'unsigned-byte))) 184 (loop for i from 0 below vlen do (let ((x (deref v i))) (setf (aref rval i) x))) 185 (is (string= (octets-to-string val) (concatenate 'string (map 'vector #'code-char rval))))) 186 (rocksdb-delete db wopts k klen errptr) 187 (is (null-alien errptr)) 188 (rocksdb-writeoptions-destroy wopts) 189 (rocksdb-readoptions-destroy ropts) 190 (rocksdb-cancel-all-background-work db nil) 192 (rocksdb-destroy-db opts path errptr) 193 (is (null-alien errptr)) 194 (rocksdb-options-destroy opts)))) 197 "Test SST file write/ingest functionality." 198 (let* ((opts (test-opts)) 199 (path (rocksdb-test-dir)) 200 (file (rocksdb-test-file)) 201 (db (rocksdb-open opts path nil)) 206 (eopts (rocksdb-envoptions-create)) 207 (iopts (rocksdb-ingestexternalfileoptions-create)) 208 (ropts (rocksdb-readoptions-create)) 209 (writer (rocksdb-sstfilewriter-create eopts opts))) 210 (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen)) 211 (v (* unsigned-char) (make-alien unsigned-char vlen)) 212 (flist (array c-string 1)) 213 (errptr rocksdb-errptr nil)) 218 (setf (deref flist 0) file) 220 (rocksdb-sstfilewriter-open writer file errptr) 221 ;; insert rows into sst file 222 (rocksdb-sstfilewriter-put writer k klen v vlen errptr) 223 (is (null-alien errptr)) 224 (rocksdb-sstfilewriter-finish writer errptr) 225 (is (null-alien errptr)) 227 (rocksdb-ingest-external-file db (cast flist (* c-string)) 1 iopts errptr) 228 (is (null-alien errptr)) 229 (is (string= (octets-to-string val) (cast (rocksdb-get db ropts k klen (make-alien size-t vlen) errptr) c-string))) 231 ;; rocksdb-sstfilewriter-file-size 232 (rocksdb-sstfilewriter-destroy writer) 234 (rocksdb-destroy-db opts path errptr) 235 (rocksdb-options-destroy opts) 236 (rocksdb-envoptions-destroy eopts) 238 (is (null-alien errptr))))) 241 "Test statistics and performance-context related functionality." 242 (rocksdb-set-perf-level (rocksdb-perf-level "enable-time-except-for-mutex")) 243 (let* ((opts (test-opts)) 244 (path (rocksdb-test-dir)) 245 (db (rocksdb-open opts path nil)) 246 (key (random-bytes 100)) 247 (val (random-bytes 100000)) 250 (wopts (rocksdb-writeoptions-create)) 251 (ctx (rocksdb::rocksdb-perfcontext-create)) 252 (hist (rocksdb-statistics-histogram-data-create))) 253 (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen)) 254 (v (* (unsigned 8)) (make-alien (unsigned 8) vlen)) 255 (errptr rocksdb-errptr nil)) 269 (debug! "stats:" (rocksdb-options-statistics-get-string opts)) 270 (rocksdb-options-statistics-get-histogram-data opts 5 hist) ;; histogram data types? uint64 somewhere 271 (debug! "count:" (rocksdb-statistics-histogram-data-get-count hist)) 272 (rocksdb-perfcontext-reset ctx) 274 (rocksdb-set-perf-level (rocksdb-perf-level "disable")) 275 (rocksdb-statistics-histogram-data-destroy hist) 277 (rocksdb-destroy-db opts path errptr) 278 (rocksdb-options-destroy opts)))) 280 ;; stats-dump-period-sec 283 "Test BlobDB functionality." 284 (let* ((opts (test-opts)) 285 (path (rocksdb-test-dir)) 287 (key (random-bytes 8)) 288 (val (make-array 9999 :initial-element 36)) 291 (wopts (rocksdb-writeoptions-create)) 292 (ropts (rocksdb-readoptions-create)) 293 (bcache (rocksdb-cache-create-lru 128))) 294 (rocksdb-options-set-enable-blob-files opts t) 295 (rocksdb-options-set-enable-blob-gc opts t) 296 (rocksdb-options-set-blob-compression-type opts (rocksdb-compression-backend "zstd")) 297 (rocksdb-options-set-blob-cache opts bcache) 298 (setf db (rocksdb-open opts path nil)) 300 (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen)) 301 (v (* (unsigned 8)) (make-alien (unsigned 8) vlen)) 302 (errptr rocksdb-errptr nil)) 303 (debug! "min blob file size: " (rocksdb-options-get-min-blob-size opts)) 304 (debug! "max blob file size: " (rocksdb-options-get-blob-file-size opts)) 318 (is (null-alien errptr)) 319 (rocksdb:rocksdb-flush db (rocksdb-flushoptions-create) errptr) 320 (is (null-alien errptr)) 327 (make-alien size-t vlen) 330 (rocksdb-writeoptions-destroy wopts) 331 (rocksdb-readoptions-destroy ropts) 333 (rocksdb-destroy-db opts path errptr) 334 (is (null-alien errptr)) 335 (rocksdb-options-destroy opts)))) 337 (deftest transaction () 338 "Test simple transactions using both TransactionDB and OptimisticTransactionDB." 339 (let* ((opts (test-opts)) 340 (path (rocksdb-test-dir)) 341 (db (rocksdb-open opts path nil)) 346 (wopts (rocksdb-writeoptions-create)) 347 (ropts (rocksdb-readoptions-create))) 348 (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen)) 349 (v (* (unsigned 8)) (make-alien (unsigned 8) vlen)) 350 (errptr rocksdb-errptr nil)) 356 (rocksdb-writeoptions-destroy wopts) 357 (rocksdb-readoptions-destroy ropts) 359 (rocksdb-destroy-db opts path errptr) 360 (rocksdb-options-destroy opts) 361 (is (null-alien errptr))))) 364 "Test metadata functionality :: cf-meta -> level-meta -> sst-file-meta" 367 (deftest properties () 368 "Test the ROCKSDB-GET-PROPERTY-* functions." 369 ;; *rocksdb-properties* 370 (let* ((opts (test-opts)) 371 (path (rocksdb-test-dir)) 372 (db (rocksdb-open opts path nil)) 377 (wopts (rocksdb-writeoptions-create)) 378 (ropts (rocksdb-readoptions-create))) 379 (is (stringp (debug! (rocksdb-property-value db (make-alien-string "rocksdb.stats"))))) 380 (is (zerop (parse-integer (rocksdb-property-value db (make-alien-string "rocksdb.num-files-at-level3"))))))) 382 (define-merge-operator dummy nil 387 "Test low-level merge-operator functionality using Alien Callbacks." 388 (is (with-alien ((k (array unsigned-char)) 389 (v (array unsigned-char)) 390 (ops (array (array unsigned-char))) 391 (s (array unsigned-char))) 393 (alien-callable-function 394 'rocksdb-concat-full-merge) 395 k 0 v 0 ops (make-alien size-t 0) 0 s (make-alien size-t 0)))) 398 (with-alien ((k (array unsigned-char)) 399 (ops (array (array unsigned-char))) 400 (s (array unsigned-char))) 402 (alien-callable-function 403 'rocksdb-concat-partial-merge) 404 k 0 ops (make-alien size-t 0) 0 s (make-alien size-t 0))))) 405 (alien-callable-function 'rocksdb-concat-full-merge) 406 (alien-callable-function 'rocksdb-concat-partial-merge) 409 (string-trim "rocksdb:" (alien-funcall (alien-callable-function 'rocksdb-name)))))) 411 (with-alien ((str c-string (make-alien-string "")) 413 (is (null (alien-funcall (alien-callable-function 'rocksdb-delete-value) state str 1)))) 415 (is (null (alien-funcall (alien-callable-function 'rocksdb-destructor) (make-alien (* t))))) 417 (with-alien ((state (* t)) 418 (destructor (* rocksdb-destructor-function)) 419 (full-merge (* rocksdb-full-merge-function)) 420 (partial-merge (* rocksdb-partial-merge-function)) 421 (delete-value (* rocksdb-delete-value-function)) 422 (name (* rocksdb-name-function))) 423 (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name) 424 '(alien (* rocksdb-mergeoperator))))) 426 (with-alien ((state (* t)) 427 (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor))) 428 (full-merge (* rocksdb-full-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-full-merge))) 429 (partial-merge (* rocksdb-partial-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-partial-merge))) 430 (delete-value (* rocksdb-delete-value-function) (alien-sap (alien-callable-function 'rocksdb-delete-value))) 431 (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-concat-merge-name)))) 432 (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name) 433 '(alien (* rocksdb-mergeoperator))))) 434 (with-opt (o (test-opts) nil) 435 (rocksdb-options-set-merge-operator o (create-dummy-mergeoperator)) 437 ;; merge_operator=DUMMY in OPTIONS file 441 (deftest comparator () 442 "Test low-level comparator API." 443 (with-alien ((state (* t)) 444 (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor))) 445 (compare (* rocksdb-compare-function) (alien-sap (alien-callable-function 'rocksdb-compare-never))) 446 (compare-with-ts (* rocksdb-compare-with-ts-function)) 447 (compare-without-ts (* rocksdb-compare-without-ts-function)) 448 (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-name)))) 449 (is (typep (rocksdb-comparator-create state destructor compare name) 450 '(alien (* rocksdb-comparator)))) 451 (is (typep (rocksdb-comparator-with-ts-create state destructor compare compare-with-ts compare-without-ts name) 452 '(alien (* rocksdb-comparator)))) 453 ;; TODO - need to test with column-family options 454 (with-opt (o (test-opts) nil) 455 (rocksdb-options-set-comparator 457 (rocksdb-comparator-create state destructor compare name))))) 459 (deftest compaction () 460 "Test low-level compactionfilter API." 461 (with-alien ((state (* t)) 462 (context (* rocksdb-compactionfiltercontext))) 464 (rocksdb-compactionfilter-create state 465 (alien-sap (alien-callable-function 'rocksdb-destructor)) 466 (alien-sap (alien-callable-function 'rocksdb-filter-never)) 467 (alien-sap (alien-callable-function 'rocksdb-name))) 468 '(alien (* rocksdb-compactionfilter)))) 470 (rocksdb-compactionfilterfactory-create state 471 (alien-sap (alien-callable-function 'rocksdb-destructor)) 472 (alien-sap (alien-callable-function 473 'rocksdb-create-compaction-filter-never)) 474 (alien-sap (alien-callable-function 'rocksdb-name))) 475 '(alien (* rocksdb-compactionfilterfactory))))) 478 (with-opt (o (test-opts) nil) 483 "Test logging functionality." 484 (with-alien ((state (* t)) 487 (log (* rocksdb-log-function) (alien-sap (alien-callable-function 'rocksdb-log-default)))) 489 (rocksdb-logger-create-stderr-logger lev msg) 490 '(alien (* rocksdb-logger)))) 492 (rocksdb-logger-create-callback-logger lev log state) 493 '(alien (* rocksdb-logger)))))) 495 (deftest writebatch () 496 "Test writebatch functionality." 497 (with-alien ((fput (* rocksdb-put-function)) 498 (fdeleted (* rocksdb-deleted-function)) 499 (fdeleted-cf (* rocksdb-deleted-cf-function)) 500 (fput-cf (* rocksdb-put-cf-function)) 501 (fmerge-cf (* rocksdb-merge-cf-function)) 502 (fget-ts (* rocksdb-get-ts-size-function))) 504 (rocksdb-writebatch-create) 505 '(alien (* rocksdb-writebatch)))) 507 (rocksdb-writebatch-wi-create 0 0) 508 '(alien (* rocksdb-writebatch-wi)))))) 510 (deftest slicetransform () 511 "Test slicetransform functionality." 512 (with-alien ((state (* t)) 513 (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor))) 514 (transform (* rocksdb-transform-function)) 515 (in-domain (* rocksdb-in-domain-function)) 516 (in-range (* rocksdb-in-range-function)) 517 (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-name)))) 519 (rocksdb-slicetransform-create state destructor transform in-domain in-range name) 520 '(alien (* rocksdb-slicetransform))))))