Mercurial > core / lisp/ffi/rocksdb/tests.lisp
changeset 616: |
c48704d7b06f |
parent: |
57813b8ee029
|
child: |
5e57683a0c28 |
author: |
Richard Westhaver <ellis@rwest.io> |
date: |
Tue, 20 Aug 2024 21:02:58 -0400 |
permissions: |
-rw-r--r-- |
description: |
writebatch callbacks and fixes |
1 ;;; rocksdb/tests.lisp --- RocksDB tests 4 (defpackage :rocksdb/tests 5 (:use :cl :std :rt :rocksdb :sb-ext :sb-alien :log)) 7 (in-package :rocksdb/tests) 15 (defun rocksdb-test-dir () 16 (format nil "/tmp/~A/" (gensym "rocksdb-tests-"))) 18 (defun rocksdb-test-file () 19 (format nil "/tmp/~A" (gensym "rocksdb-test-"))) 22 (let ((default (rocksdb-options-create))) 23 (rocksdb-options-set-create-if-missing default t) 26 ;; not thread safe (gensym-counter) 27 (defun genkey (&optional prefix) (string-to-octets (symbol-name (gensym (or prefix "key"))))) 28 (defun genval (&optional prefix) (string-to-octets (symbol-name (gensym (or prefix "val"))))) 30 (defmacro with-opt ((var create destroy) &body body) 31 `(let ((,var ,create)) 32 (unwind-protect (progn ,@body) 36 (with-opt (o (rocksdb-options-create) (rocksdb-options-destroy o)) 38 (rocksdb-options-set-create-if-missing o t) 39 (rocksdb-options-get-create-if-missing o) 40 (rocksdb-options-set-create-missing-column-families o t) 41 (rocksdb-options-get-create-missing-column-families o) 42 (rocksdb-options-set-error-if-exists o t) 43 (rocksdb-options-get-error-if-exists o) 44 (rocksdb-options-set-paranoid-checks o t) 45 (rocksdb-options-get-paranoid-checks o) 46 (rocksdb-options-set-compression-options-use-zstd-dict-trainer o t) 47 (rocksdb-options-get-compression-options-use-zstd-dict-trainer o) 48 (rocksdb-options-set-enable-blob-gc o t) 49 (rocksdb-options-get-enable-blob-gc o) 50 (rocksdb-options-set-allow-ingest-behind o t) 51 (rocksdb-options-get-allow-ingest-behind o) 52 (rocksdb-options-set-skip-stats-update-on-db-open o t) 53 (rocksdb-options-get-skip-stats-update-on-db-open o) 54 (rocksdb-options-set-skip-checking-sst-file-sizes-on-db-open o t) 55 (rocksdb-options-get-skip-checking-sst-file-sizes-on-db-open o) 56 (rocksdb-options-set-enable-blob-files o t) 57 (rocksdb-options-get-enable-blob-files o) 58 (rocksdb-options-set-enable-pipelined-write o t) 59 (rocksdb-options-get-enable-pipelined-write o) 60 (rocksdb-options-set-unordered-write o t) 61 (rocksdb-options-get-unordered-write o) 62 (rocksdb-options-set-allow-mmap-reads o t) 63 (rocksdb-options-get-allow-mmap-reads o) 64 (rocksdb-options-set-allow-mmap-writes o t) 65 (rocksdb-options-get-allow-mmap-writes o) 66 (rocksdb-options-set-use-direct-reads o t) 67 (rocksdb-options-get-use-direct-reads o) 68 (rocksdb-options-set-use-direct-io-for-flush-and-compaction o t) 69 (rocksdb-options-get-use-direct-io-for-flush-and-compaction o) 70 (rocksdb-options-set-is-fd-close-on-exec o t) 71 (rocksdb-options-get-is-fd-close-on-exec o) 72 (rocksdb-options-set-inplace-update-support o t) 73 (rocksdb-options-get-inplace-update-support o) 74 (rocksdb-options-set-advise-random-on-open o t) 75 (rocksdb-options-get-advise-random-on-open o) 76 (rocksdb-options-set-atomic-flush o t) 77 (rocksdb-options-get-atomic-flush o) 78 (rocksdb-options-set-manual-wal-flush o t) 79 (rocksdb-options-get-manual-wal-flush o) 80 (rocksdb-options-set-avoid-unnecessary-blocking-io o t) 81 (rocksdb-options-get-avoid-unnecessary-blocking-io o) 82 ;; this is full-width value 0-255, not boolean 83 (rocksdb-options-set-level-compaction-dynamic-level-bytes o 20) 84 (rocksdb-options-get-level-compaction-dynamic-level-bytes o) 86 (rocksdb-options-set-compression-options-parallel-threads o 4) 87 (rocksdb-options-get-compression-options-parallel-threads o) 88 (rocksdb-options-set-info-log-level o 1) 89 (rocksdb-options-get-info-log-level o) 90 (rocksdb-options-set-max-open-files o 100) 91 (rocksdb-options-get-max-open-files o) 92 (rocksdb-options-set-max-file-opening-threads o 4) 93 (rocksdb-options-get-max-file-opening-threads o) 94 (rocksdb-options-set-compression-options-zstd-max-train-bytes o 1024) 95 (rocksdb-options-get-compression-options-zstd-max-train-bytes o) 96 (rocksdb-options-set-num-levels o 4) 97 (rocksdb-options-get-num-levels o) 98 (rocksdb-options-set-level0-file-num-compaction-trigger o 16) 99 (rocksdb-options-get-level0-file-num-compaction-trigger o) 100 (rocksdb-options-set-level0-slowdown-writes-trigger o 1024) 101 (rocksdb-options-get-level0-slowdown-writes-trigger o) 102 (rocksdb-options-set-level0-stop-writes-trigger o 1024) 103 (rocksdb-options-get-level0-stop-writes-trigger o) 104 (rocksdb-options-set-target-file-size-multiplier o 4) 105 (rocksdb-options-get-target-file-size-multiplier o) 107 (rocksdb-options-set-write-buffer-size o 1024) 108 (rocksdb-options-get-write-buffer-size o) 109 (rocksdb-options-set-db-write-buffer-size o 1024) 110 (rocksdb-options-get-db-write-buffer-size o) 112 (rocksdb-options-set-compression-options-max-dict-buffer-bytes o 1024) 113 (rocksdb-options-get-compression-options-max-dict-buffer-bytes o) 114 (rocksdb-options-set-max-total-wal-size o 1024) 115 (rocksdb-options-get-max-total-wal-size o) 116 (rocksdb-options-set-target-file-size-base o 1024) 117 (rocksdb-options-get-target-file-size-base o) 118 (rocksdb-options-set-max-bytes-for-level-base o 1024) 119 (rocksdb-options-get-max-bytes-for-level-base o) 121 ;; (rocksdb-options-set-max-bytes-for-level-multiplier o (the double-float (/ 1 3))) 123 (let ((opts (rocksdb-options-create)) 124 (wopts (rocksdb-writeoptions-create)) 125 (ropts (rocksdb-readoptions-create)) 126 (bopts (rocksdb-block-based-options-create))) 127 (rocksdb-options-set-create-if-missing opts t) 129 (rocksdb-options-destroy opts) 130 (rocksdb-writeoptions-destroy wopts) 131 (rocksdb-readoptions-destroy ropts) 132 (rocksdb-block-based-options-destroy bopts))) 134 (defun make-errptr () 135 (make-alien rocksdb-errptr)) 138 "Test basic RocksDB functionality. Inserts KV pair into a temporary 139 DB where K and V are both Lisp strings." 140 (let* ((opts (test-opts)) 141 (path (rocksdb-test-dir)) 142 (db (rocksdb-open opts path nil)) 147 (wopts (rocksdb-writeoptions-create)) 148 (ropts (rocksdb-readoptions-create))) 149 (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen)) 150 (v (* unsigned-char) (make-alien unsigned-char vlen)) 151 (errptr rocksdb-errptr nil)) 164 (is (null-alien errptr)) 165 ;; get V from DB given K 166 (rocksdb:rocksdb-cancel-all-background-work db t) 167 (rocksdb-get db ropts k klen (make-alien size-t vlen) errptr) 168 (is (null-alien errptr)) 169 ;; copy V to RVAL and validate 170 (let ((rval (make-array vlen :element-type 'unsigned-byte))) 171 (loop for i from 0 below vlen do (let ((x (deref v i))) (setf (aref rval i) x))) 172 (is (string= (octets-to-string val) (concatenate 'string (map 'vector #'code-char rval))))) 173 (rocksdb-delete db wopts k klen errptr) 174 (is (null-alien errptr)) 175 (rocksdb-writeoptions-destroy wopts) 176 (rocksdb-readoptions-destroy ropts) 177 (rocksdb-cancel-all-background-work db nil) 179 (rocksdb-destroy-db opts path errptr) 180 (is (null-alien errptr)) 181 (rocksdb-options-destroy opts)))) 184 "Test SST file write/ingest functionality." 185 (let* ((opts (test-opts)) 186 (path (rocksdb-test-dir)) 187 (file (rocksdb-test-file)) 188 (db (rocksdb-open opts path nil)) 193 (eopts (rocksdb-envoptions-create)) 194 (iopts (rocksdb-ingestexternalfileoptions-create)) 195 (ropts (rocksdb-readoptions-create)) 196 (writer (rocksdb-sstfilewriter-create eopts opts))) 197 (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen)) 198 (v (* unsigned-char) (make-alien unsigned-char vlen)) 199 (flist (array c-string 1)) 200 (errptr rocksdb-errptr nil)) 205 (setf (deref flist 0) file) 207 (rocksdb-sstfilewriter-open writer file errptr) 208 ;; insert rows into sst file 209 (rocksdb-sstfilewriter-put writer k klen v vlen errptr) 210 (is (null-alien errptr)) 211 (rocksdb-sstfilewriter-finish writer errptr) 212 (is (null-alien errptr)) 214 (rocksdb-ingest-external-file db (cast flist (* c-string)) 1 iopts errptr) 215 (is (null-alien errptr)) 216 (let ((vres (make-array vlen :element-type 'octet :fill-pointer 0))) 217 (is (string= (octets-to-string val) (cast (rocksdb-get db ropts k klen (make-alien size-t vlen) errptr) c-string)))) 219 ;; rocksdb-sstfilewriter-file-size 220 (rocksdb-sstfilewriter-destroy writer) 222 (rocksdb-destroy-db opts path errptr) 223 (rocksdb-options-destroy opts) 224 (rocksdb-envoptions-destroy eopts) 226 (is (null-alien errptr))))) 229 "Test statistics and performance-context related functionality." 230 (rocksdb-set-perf-level (rocksdb-perf-level "enable-time-except-for-mutex")) 231 (let* ((opts (test-opts)) 232 (path (rocksdb-test-dir)) 233 (db (rocksdb-open opts path nil)) 234 (key (random-bytes 100)) 235 (val (random-bytes 100000)) 238 (wopts (rocksdb-writeoptions-create)) 239 (ropts (rocksdb-readoptions-create)) 240 (ctx (rocksdb::rocksdb-perfcontext-create)) 241 (hist (rocksdb-statistics-histogram-data-create))) 242 (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen)) 243 (v (* (unsigned 8)) (make-alien (unsigned 8) vlen)) 244 (errptr rocksdb-errptr nil)) 258 (debug! "stats:" (rocksdb-options-statistics-get-string opts)) 259 (rocksdb-options-statistics-get-histogram-data opts 5 hist) ;; histogram data types? uint64 somewhere 260 (debug! "count:" (rocksdb-statistics-histogram-data-get-count hist)) 261 (rocksdb-perfcontext-reset ctx) 263 (rocksdb-set-perf-level (rocksdb-perf-level "disable")) 264 (rocksdb-statistics-histogram-data-destroy hist) 266 (rocksdb-destroy-db opts path errptr) 267 (rocksdb-options-destroy opts)))) 269 ;; stats-dump-period-sec 272 "Test BlobDB functionality." 273 (let* ((opts (test-opts)) 274 (path (rocksdb-test-dir)) 276 (key (random-bytes 8)) 277 (val (make-array 9999 :initial-element 36)) 280 (wopts (rocksdb-writeoptions-create)) 281 (ropts (rocksdb-readoptions-create)) 282 (bcache (rocksdb-cache-create-lru 128))) 283 (rocksdb-options-set-enable-blob-files opts t) 284 (rocksdb-options-set-enable-blob-gc opts t) 285 (rocksdb-options-set-blob-compression-type opts (rocksdb-compression-backend "zstd")) 286 (rocksdb-options-set-blob-cache opts bcache) 287 (setf db (rocksdb-open opts path nil)) 289 (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen)) 290 (v (* (unsigned 8)) (make-alien (unsigned 8) vlen)) 291 (errptr rocksdb-errptr nil)) 292 (debug! "min blob file size: " (rocksdb-options-get-min-blob-size opts)) 293 (debug! "max blob file size: " (rocksdb-options-get-blob-file-size opts)) 307 (is (null-alien errptr)) 308 (rocksdb:rocksdb-flush db (rocksdb-flushoptions-create) errptr) 309 (is (null-alien errptr)) 316 (make-alien size-t vlen) 319 (rocksdb-writeoptions-destroy wopts) 320 (rocksdb-readoptions-destroy ropts) 322 (rocksdb-destroy-db opts path errptr) 323 (is (null-alien errptr)) 324 (rocksdb-options-destroy opts)))) 326 (deftest transaction () 327 "Test simple transactions using both TransactionDB and OptimisticTransactionDB." 328 (let* ((opts (test-opts)) 329 (path (rocksdb-test-dir)) 330 (db (rocksdb-open opts path nil)) 335 (wopts (rocksdb-writeoptions-create)) 336 (ropts (rocksdb-readoptions-create))) 337 (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen)) 338 (v (* (unsigned 8)) (make-alien (unsigned 8) vlen)) 339 (errptr rocksdb-errptr nil)) 345 (rocksdb-writeoptions-destroy wopts) 346 (rocksdb-readoptions-destroy ropts) 348 (rocksdb-destroy-db opts path errptr) 349 (rocksdb-options-destroy opts) 350 (is (null-alien errptr))))) 353 "Test metadata functionality :: cf-meta -> level-meta -> sst-file-meta" 356 (deftest properties () 357 "Test the ROCKSDB-GET-PROPERTY-* functions." 358 ;; *rocksdb-properties* 359 (let* ((opts (test-opts)) 360 (path (rocksdb-test-dir)) 361 (db (rocksdb-open opts path nil)) 366 (wopts (rocksdb-writeoptions-create)) 367 (ropts (rocksdb-readoptions-create))) 368 (is (stringp (debug! (rocksdb-property-value db (make-alien-string "rocksdb.stats"))))) 369 (is (zerop (parse-integer (rocksdb-property-value db (make-alien-string "rocksdb.num-files-at-level3"))))))) 372 "Test low-level merge-operator functionality using Alien Callbacks." 373 (is (with-alien ((k (array unsigned-char)) 374 (v (array unsigned-char)) 375 (ops (array (array unsigned-char))) 376 (s (array unsigned-char))) 378 (alien-callable-function 379 'rocksdb-concat-full-merge) 380 k 0 v 0 ops (make-alien size-t 0) 0 s (make-alien size-t 0)))) 383 (with-alien ((k (array unsigned-char)) 384 (ops (array (array unsigned-char))) 385 (s (array unsigned-char))) 387 (alien-callable-function 388 'rocksdb-concat-partial-merge) 389 k 0 ops (make-alien size-t 0) 0 s (make-alien size-t 0))))) 390 (alien-callable-function 'rocksdb-concat-full-merge) 391 (alien-callable-function 'rocksdb-concat-partial-merge) 394 (string-trim "rocksdb:" (alien-funcall (alien-callable-function 'rocksdb-name)))))) 396 (with-alien ((str c-string (make-alien-string "")) 398 (is (null (alien-funcall (alien-callable-function 'rocksdb-delete-value) state str 1)))) 400 (is (null (alien-funcall (alien-callable-function 'rocksdb-destructor) (make-alien (* t))))) 403 (with-alien ((state (* t)) 404 (destructor (* rocksdb-destructor-function)) 405 (full-merge (* rocksdb-full-merge-function)) 406 (partial-merge (* rocksdb-partial-merge-function)) 407 (delete-value (* rocksdb-delete-value-function)) 408 (name (* rocksdb-name-function))) 409 (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name) 410 '(alien (* rocksdb-mergeoperator))))) 413 (with-alien ((state (* t)) 414 (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor))) 415 (full-merge (* rocksdb-full-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-full-merge))) 416 (partial-merge (* rocksdb-partial-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-partial-merge))) 417 (delete-value (* rocksdb-delete-value-function) (alien-sap (alien-callable-function 'rocksdb-delete-value))) 418 (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-concat-merge-name)))) 419 (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name) 420 '(alien (* rocksdb-mergeoperator)))))) 422 (deftest comparator () 423 "Test low-level comparator API." 424 (with-alien ((state (* t)) 425 (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor))) 426 (compare (* rocksdb-compare-function) (alien-sap (alien-callable-function 'rocksdb-compare-never))) 427 (compare-with-ts (* rocksdb-compare-with-ts-function)) 428 (compare-without-ts (* rocksdb-compare-without-ts-function)) 429 (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-name)))) 430 (is (typep (rocksdb-comparator-create state destructor compare name) 431 '(alien (* rocksdb-comparator)))) 432 (is (typep (rocksdb-comparator-with-ts-create state destructor compare compare-with-ts compare-without-ts name) 433 '(alien (* rocksdb-comparator)))))) 435 (deftest compaction () 436 "Test low-level compactionfilter API." 437 (with-alien ((state (* t)) 438 (context (* rocksdb-compactionfiltercontext))) 440 (rocksdb-compactionfilter-create state 441 (alien-sap (alien-callable-function 'rocksdb-destructor)) 442 (alien-sap (alien-callable-function 'rocksdb-filter-never)) 443 (alien-sap (alien-callable-function 'rocksdb-name))) 444 '(alien (* rocksdb-compactionfilter)))) 446 (rocksdb-compactionfilterfactory-create state 447 (alien-sap (alien-callable-function 'rocksdb-destructor)) 448 (alien-sap (alien-callable-function 449 'rocksdb-create-compaction-filter-never)) 450 (alien-sap (alien-callable-function 'rocksdb-name))) 451 '(alien (* rocksdb-compactionfilterfactory)))))) 454 "Test logging functionality." 455 (with-alien ((state (* t)) 458 (log (* rocksdb-log-function) (alien-sap (alien-callable-function 'rocksdb-log-default)))) 460 (rocksdb-logger-create-stderr-logger lev msg) 461 '(alien (* rocksdb-logger)))) 463 (rocksdb-logger-create-callback-logger lev log state) 464 '(alien (* rocksdb-logger)))))) 466 (deftest writebatch () 467 "Test writebatch functionality." 468 (with-alien ((fput (* rocksdb-put-function)) 469 (fdeleted (* rocksdb-deleted-function)) 470 (fdeleted-cf (* rocksdb-deleted-cf-function)) 471 (fput-cf (* rocksdb-put-cf-function)) 472 (fmerge-cf (* rocksdb-merge-cf-function)) 473 (fget-ts (* rocksdb-get-ts-size-function))) 475 (rocksdb-writebatch-create) 476 '(alien (* rocksdb-writebatch)))) 478 (rocksdb-writebatch-wi-create 0 0) 479 '(alien (* rocksdb-writebatch-wi)))))) 481 (deftest slicetransform (:skip t) 482 "Test slicetransform functionality." 483 (with-alien ((state (* t)) 484 (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor))) 485 (transform (* t) (* rocksdb-transform-function)) 486 (in-domain (* rocksdb-in-domain-function)) 487 (in-range (* rocksdb-in-range-function)) 488 (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-name)))) 489 (rocksdb-slicetransform-create state destructor transform in-domain in-range name)))