changelog shortlog graph tags branches changeset files revisions annotate raw help

Mercurial > core / lisp/ffi/rocksdb/tests.lisp

changeset 623: a304c9713a51
parent: 5e57683a0c28
author: Richard Westhaver <ellis@rwest.io>
date: Sun, 25 Aug 2024 00:14:17 -0400
permissions: -rw-r--r--
description: init graph.el
1 ;;; rocksdb/tests.lisp --- RocksDB tests
2 
3 ;;; Code:
4 (defpackage :rocksdb/tests
5  (:use :cl :std :rt :rocksdb :sb-ext :sb-alien :log))
6 
7 (in-package :rocksdb/tests)
8 
9 (defsuite :rocksdb)
10 (in-suite :rocksdb)
11 
12 (load-rocksdb)
13 (init-log-timestamp)
14 
15 (defun make-errptr ()
16  (make-alien rocksdb-errptr))
17 
18 (defun rocksdb-test-dir ()
19  (format nil "/tmp/~A/" (gensym "rocksdb-tests-")))
20 
21 (defun rocksdb-test-file ()
22  (format nil "/tmp/~A" (gensym "rocksdb-test-")))
23 
24 (defun test-opts ()
25  (let ((default (rocksdb-options-create)))
26  (rocksdb-options-set-create-if-missing default t)
27  default))
28 
29 (defmacro with-errptr (sym &body body)
30  `(with-alien ((,sym rocksdb-errptr (make-errptr)))
31  ,@body))
32 
33 (defmacro with-temp-db (sym (&optional (opts (test-opts)) (path (rocksdb-test-dir))) &body body)
34  `(with-errptr err
35  (let* ((opts ,opts)
36  (path ,path)
37  (,sym (rocksdb-open opts path err)))
38  (unwind-protect
39  (progn ,@body)
40  (rocksdb-close ,sym)))))
41 
42 ;; not thread safe (gensym-counter)
43 (defun genkey (&optional prefix) (string-to-octets (symbol-name (gensym (or prefix "key")))))
44 (defun genval (&optional prefix) (string-to-octets (symbol-name (gensym (or prefix "val")))))
45 
46 (defmacro with-opt ((var create destroy) &body body)
47  `(let ((,var ,create))
48  (unwind-protect (progn ,@body)
49  ,destroy)))
50 
51 (deftest opts ()
52  (with-opt (o (rocksdb-options-create) (rocksdb-options-destroy o))
53  ;; unsigned-char
54  (rocksdb-options-set-create-if-missing o t)
55  (rocksdb-options-get-create-if-missing o)
56  (rocksdb-options-set-create-missing-column-families o t)
57  (rocksdb-options-get-create-missing-column-families o)
58  (rocksdb-options-set-error-if-exists o t)
59  (rocksdb-options-get-error-if-exists o)
60  (rocksdb-options-set-paranoid-checks o t)
61  (rocksdb-options-get-paranoid-checks o)
62  (rocksdb-options-set-compression-options-use-zstd-dict-trainer o t)
63  (rocksdb-options-get-compression-options-use-zstd-dict-trainer o)
64  (rocksdb-options-set-enable-blob-gc o t)
65  (rocksdb-options-get-enable-blob-gc o)
66  (rocksdb-options-set-allow-ingest-behind o t)
67  (rocksdb-options-get-allow-ingest-behind o)
68  (rocksdb-options-set-skip-stats-update-on-db-open o t)
69  (rocksdb-options-get-skip-stats-update-on-db-open o)
70  (rocksdb-options-set-skip-checking-sst-file-sizes-on-db-open o t)
71  (rocksdb-options-get-skip-checking-sst-file-sizes-on-db-open o)
72  (rocksdb-options-set-enable-blob-files o t)
73  (rocksdb-options-get-enable-blob-files o)
74  (rocksdb-options-set-enable-pipelined-write o t)
75  (rocksdb-options-get-enable-pipelined-write o)
76  (rocksdb-options-set-unordered-write o t)
77  (rocksdb-options-get-unordered-write o)
78  (rocksdb-options-set-allow-mmap-reads o t)
79  (rocksdb-options-get-allow-mmap-reads o)
80  (rocksdb-options-set-allow-mmap-writes o t)
81  (rocksdb-options-get-allow-mmap-writes o)
82  (rocksdb-options-set-use-direct-reads o t)
83  (rocksdb-options-get-use-direct-reads o)
84  (rocksdb-options-set-use-direct-io-for-flush-and-compaction o t)
85  (rocksdb-options-get-use-direct-io-for-flush-and-compaction o)
86  (rocksdb-options-set-is-fd-close-on-exec o t)
87  (rocksdb-options-get-is-fd-close-on-exec o)
88  (rocksdb-options-set-inplace-update-support o t)
89  (rocksdb-options-get-inplace-update-support o)
90  (rocksdb-options-set-advise-random-on-open o t)
91  (rocksdb-options-get-advise-random-on-open o)
92  (rocksdb-options-set-atomic-flush o t)
93  (rocksdb-options-get-atomic-flush o)
94  (rocksdb-options-set-manual-wal-flush o t)
95  (rocksdb-options-get-manual-wal-flush o)
96  (rocksdb-options-set-avoid-unnecessary-blocking-io o t)
97  (rocksdb-options-get-avoid-unnecessary-blocking-io o)
98  ;; this is full-width value 0-255, not boolean
99  (rocksdb-options-set-level-compaction-dynamic-level-bytes o 20)
100  (rocksdb-options-get-level-compaction-dynamic-level-bytes o)
101  ;; int
102  (rocksdb-options-set-compression-options-parallel-threads o 4)
103  (rocksdb-options-get-compression-options-parallel-threads o)
104  (rocksdb-options-set-info-log-level o 1)
105  (rocksdb-options-get-info-log-level o)
106  (rocksdb-options-set-max-open-files o 100)
107  (rocksdb-options-get-max-open-files o)
108  (rocksdb-options-set-max-file-opening-threads o 4)
109  (rocksdb-options-get-max-file-opening-threads o)
110  (rocksdb-options-set-compression-options-zstd-max-train-bytes o 1024)
111  (rocksdb-options-get-compression-options-zstd-max-train-bytes o)
112  (rocksdb-options-set-num-levels o 4)
113  (rocksdb-options-get-num-levels o)
114  (rocksdb-options-set-level0-file-num-compaction-trigger o 16)
115  (rocksdb-options-get-level0-file-num-compaction-trigger o)
116  (rocksdb-options-set-level0-slowdown-writes-trigger o 1024)
117  (rocksdb-options-get-level0-slowdown-writes-trigger o)
118  (rocksdb-options-set-level0-stop-writes-trigger o 1024)
119  (rocksdb-options-get-level0-stop-writes-trigger o)
120  (rocksdb-options-set-target-file-size-multiplier o 4)
121  (rocksdb-options-get-target-file-size-multiplier o)
122  ;; size-t
123  (rocksdb-options-set-write-buffer-size o 1024)
124  (rocksdb-options-get-write-buffer-size o)
125  (rocksdb-options-set-db-write-buffer-size o 1024)
126  (rocksdb-options-get-db-write-buffer-size o)
127  ;; unsigned-long
128  (rocksdb-options-set-compression-options-max-dict-buffer-bytes o 1024)
129  (rocksdb-options-get-compression-options-max-dict-buffer-bytes o)
130  (rocksdb-options-set-max-total-wal-size o 1024)
131  (rocksdb-options-get-max-total-wal-size o)
132  (rocksdb-options-set-target-file-size-base o 1024)
133  (rocksdb-options-get-target-file-size-base o)
134  (rocksdb-options-set-max-bytes-for-level-base o 1024)
135  (rocksdb-options-get-max-bytes-for-level-base o)
136  ;; double
137  ;; (rocksdb-options-set-max-bytes-for-level-multiplier o (the double-float (/ 1 3)))
138  )
139  (let ((opts (rocksdb-options-create))
140  (wopts (rocksdb-writeoptions-create))
141  (ropts (rocksdb-readoptions-create))
142  (bopts (rocksdb-block-based-options-create)))
143  (rocksdb-options-set-create-if-missing opts t)
144  ;; cleanup
145  (rocksdb-options-destroy opts)
146  (rocksdb-writeoptions-destroy wopts)
147  (rocksdb-readoptions-destroy ropts)
148  (rocksdb-block-based-options-destroy bopts)))
149 
150 (deftest db-basic ()
151  "Test basic RocksDB functionality. Inserts KV pair into a temporary
152 DB where K and V are both Lisp strings."
153  (let* ((opts (test-opts))
154  (path (rocksdb-test-dir))
155  (db (rocksdb-open opts path nil))
156  (key (genkey))
157  (val (genval))
158  (klen (length key))
159  (vlen (length val))
160  (wopts (rocksdb-writeoptions-create))
161  (ropts (rocksdb-readoptions-create)))
162  (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen))
163  (v (* unsigned-char) (make-alien unsigned-char vlen))
164  (errptr rocksdb-errptr nil))
165  ;; copy KEY to K
166  (setfa k key)
167  ;; copy VAL to V
168  (setfa v val)
169  ;; put K:V in DB
170  (rocksdb-put db
171  wopts
172  k
173  klen
174  v
175  vlen
176  errptr)
177  (is (null-alien errptr))
178  ;; get V from DB given K
179  (rocksdb:rocksdb-cancel-all-background-work db t)
180  (rocksdb-get db ropts k klen (make-alien size-t vlen) errptr)
181  (is (null-alien errptr))
182  ;; copy V to RVAL and validate
183  (let ((rval (make-array vlen :element-type 'unsigned-byte)))
184  (loop for i from 0 below vlen do (let ((x (deref v i))) (setf (aref rval i) x)))
185  (is (string= (octets-to-string val) (concatenate 'string (map 'vector #'code-char rval)))))
186  (rocksdb-delete db wopts k klen errptr)
187  (is (null-alien errptr))
188  (rocksdb-writeoptions-destroy wopts)
189  (rocksdb-readoptions-destroy ropts)
190  (rocksdb-cancel-all-background-work db nil)
191  (rocksdb-close db)
192  (rocksdb-destroy-db opts path errptr)
193  (is (null-alien errptr))
194  (rocksdb-options-destroy opts))))
195 
196 (deftest sstfiles ()
197  "Test SST file write/ingest functionality."
198  (let* ((opts (test-opts))
199  (path (rocksdb-test-dir))
200  (file (rocksdb-test-file))
201  (db (rocksdb-open opts path nil))
202  (key (genkey))
203  (val (genval))
204  (klen (length key))
205  (vlen (length val))
206  (eopts (rocksdb-envoptions-create))
207  (iopts (rocksdb-ingestexternalfileoptions-create))
208  (ropts (rocksdb-readoptions-create))
209  (writer (rocksdb-sstfilewriter-create eopts opts)))
210  (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen))
211  (v (* unsigned-char) (make-alien unsigned-char vlen))
212  (flist (array c-string 1))
213  (errptr rocksdb-errptr nil))
214  ;; copy KEY to K
215  (setfa k key)
216  ;; copy VAL to V
217  (setfa v val)
218  (setf (deref flist 0) file)
219  ;; create writer
220  (rocksdb-sstfilewriter-open writer file errptr)
221  ;; insert rows into sst file
222  (rocksdb-sstfilewriter-put writer k klen v vlen errptr)
223  (is (null-alien errptr))
224  (rocksdb-sstfilewriter-finish writer errptr)
225  (is (null-alien errptr))
226  ;; ingest sst file
227  (rocksdb-ingest-external-file db (cast flist (* c-string)) 1 iopts errptr)
228  (is (null-alien errptr))
229  (is (string= (octets-to-string val) (cast (rocksdb-get db ropts k klen (make-alien size-t vlen) errptr) c-string)))
230 
231  ;; rocksdb-sstfilewriter-file-size
232  (rocksdb-sstfilewriter-destroy writer)
233  (rocksdb-close db)
234  (rocksdb-destroy-db opts path errptr)
235  (rocksdb-options-destroy opts)
236  (rocksdb-envoptions-destroy eopts)
237  (delete-file file)
238  (is (null-alien errptr)))))
239 
240 (deftest stats ()
241  "Test statistics and performance-context related functionality."
242  (rocksdb-set-perf-level (rocksdb-perf-level "enable-time-except-for-mutex"))
243  (let* ((opts (test-opts))
244  (path (rocksdb-test-dir))
245  (db (rocksdb-open opts path nil))
246  (key (random-bytes 100))
247  (val (random-bytes 100000))
248  (klen (length key))
249  (vlen (length val))
250  (wopts (rocksdb-writeoptions-create))
251  (ctx (rocksdb::rocksdb-perfcontext-create))
252  (hist (rocksdb-statistics-histogram-data-create)))
253  (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen))
254  (v (* (unsigned 8)) (make-alien (unsigned 8) vlen))
255  (errptr rocksdb-errptr nil))
256  ;; copy KEY to K
257  (setfa k key)
258  ;; copy VAL to V
259  (setfa v val)
260  ;; put K:V in DB
261  (rocksdb-put db
262  wopts
263  k
264  klen
265  v
266  vlen
267  errptr)
268 
269  (debug! "stats:" (rocksdb-options-statistics-get-string opts))
270  (rocksdb-options-statistics-get-histogram-data opts 5 hist) ;; histogram data types? uint64 somewhere
271  (debug! "count:" (rocksdb-statistics-histogram-data-get-count hist))
272  (rocksdb-perfcontext-reset ctx)
273  ;; ...
274  (rocksdb-set-perf-level (rocksdb-perf-level "disable"))
275  (rocksdb-statistics-histogram-data-destroy hist)
276  (rocksdb-close db)
277  (rocksdb-destroy-db opts path errptr)
278  (rocksdb-options-destroy opts))))
279 
280 ;; stats-dump-period-sec
281 
282 (deftest blob ()
283  "Test BlobDB functionality."
284  (let* ((opts (test-opts))
285  (path (rocksdb-test-dir))
286  db
287  (key (random-bytes 8))
288  (val (make-array 9999 :initial-element 36))
289  (klen (length key))
290  (vlen (length val))
291  (wopts (rocksdb-writeoptions-create))
292  (ropts (rocksdb-readoptions-create))
293  (bcache (rocksdb-cache-create-lru 128)))
294  (rocksdb-options-set-enable-blob-files opts t)
295  (rocksdb-options-set-enable-blob-gc opts t)
296  (rocksdb-options-set-blob-compression-type opts (rocksdb-compression-backend "zstd"))
297  (rocksdb-options-set-blob-cache opts bcache)
298  (setf db (rocksdb-open opts path nil))
299 
300  (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen))
301  (v (* (unsigned 8)) (make-alien (unsigned 8) vlen))
302  (errptr rocksdb-errptr nil))
303  (debug! "min blob file size: " (rocksdb-options-get-min-blob-size opts))
304  (debug! "max blob file size: " (rocksdb-options-get-blob-file-size opts))
305 
306  ;; copy KEY to K
307  (setfa k key)
308  ;; copy VAL to V
309  (setfa v val)
310  ;; put K:V in DB -
311  (rocksdb-put db
312  wopts
313  k
314  klen
315  v
316  vlen
317  errptr)
318  (is (null-alien errptr))
319  (rocksdb:rocksdb-flush db (rocksdb-flushoptions-create) errptr)
320  (is (null-alien errptr))
321  (is (stringp
322  (cast
323  (rocksdb-get db
324  ropts
325  k
326  klen
327  (make-alien size-t vlen)
328  errptr)
329  c-string)))
330  (rocksdb-writeoptions-destroy wopts)
331  (rocksdb-readoptions-destroy ropts)
332  (rocksdb-close db)
333  (rocksdb-destroy-db opts path errptr)
334  (is (null-alien errptr))
335  (rocksdb-options-destroy opts))))
336 
337 (deftest transaction ()
338  "Test simple transactions using both TransactionDB and OptimisticTransactionDB."
339  (let* ((opts (test-opts))
340  (path (rocksdb-test-dir))
341  (db (rocksdb-open opts path nil))
342  (key (genkey))
343  (val (genval))
344  (klen (length key))
345  (vlen (length val))
346  (wopts (rocksdb-writeoptions-create))
347  (ropts (rocksdb-readoptions-create)))
348  (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen))
349  (v (* (unsigned 8)) (make-alien (unsigned 8) vlen))
350  (errptr rocksdb-errptr nil))
351  ;; copy KEY to K
352  (setfa k key)
353  ;; copy VAL to V
354  (setfa v val)
355  ;; put K:V in DB -
356  (rocksdb-writeoptions-destroy wopts)
357  (rocksdb-readoptions-destroy ropts)
358  (rocksdb-close db)
359  (rocksdb-destroy-db opts path errptr)
360  (rocksdb-options-destroy opts)
361  (is (null-alien errptr)))))
362 
363 (deftest metadata ()
364  "Test metadata functionality :: cf-meta -> level-meta -> sst-file-meta"
365  nil)
366 
367 (deftest properties ()
368  "Test the ROCKSDB-GET-PROPERTY-* functions."
369  ;; *rocksdb-properties*
370  (let* ((opts (test-opts))
371  (path (rocksdb-test-dir))
372  (db (rocksdb-open opts path nil))
373  (key (genkey))
374  (val (genval))
375  (klen (length key))
376  (vlen (length val))
377  (wopts (rocksdb-writeoptions-create))
378  (ropts (rocksdb-readoptions-create)))
379  (is (stringp (debug! (rocksdb-property-value db (make-alien-string "rocksdb.stats")))))
380  (is (zerop (parse-integer (rocksdb-property-value db (make-alien-string "rocksdb.num-files-at-level3")))))))
381 
382 (define-merge-operator dummy nil
383  :full nil
384  :partial nil)
385 
386 (deftest merge ()
387  "Test low-level merge-operator functionality using Alien Callbacks."
388  (is (with-alien ((k (array unsigned-char))
389  (v (array unsigned-char))
390  (ops (array (array unsigned-char)))
391  (s (array unsigned-char)))
392  (alien-funcall
393  (alien-callable-function
394  'rocksdb-concat-full-merge)
395  k 0 v 0 ops (make-alien size-t 0) 0 s (make-alien size-t 0))))
396  (is
397  (not
398  (with-alien ((k (array unsigned-char))
399  (ops (array (array unsigned-char)))
400  (s (array unsigned-char)))
401  (alien-funcall
402  (alien-callable-function
403  'rocksdb-concat-partial-merge)
404  k 0 ops (make-alien size-t 0) 0 s (make-alien size-t 0)))))
405  (alien-callable-function 'rocksdb-concat-full-merge)
406  (alien-callable-function 'rocksdb-concat-partial-merge)
407  (is (integerp
408  (parse-integer
409  (string-trim "rocksdb:" (alien-funcall (alien-callable-function 'rocksdb-name))))))
410  ;; returns No Value
411  (with-alien ((str c-string (make-alien-string ""))
412  (state (* t)))
413  (is (null (alien-funcall (alien-callable-function 'rocksdb-delete-value) state str 1))))
414 
415  (is (null (alien-funcall (alien-callable-function 'rocksdb-destructor) (make-alien (* t)))))
416  ;; null merge op
417  (with-alien ((state (* t))
418  (destructor (* rocksdb-destructor-function))
419  (full-merge (* rocksdb-full-merge-function))
420  (partial-merge (* rocksdb-partial-merge-function))
421  (delete-value (* rocksdb-delete-value-function))
422  (name (* rocksdb-name-function)))
423  (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name)
424  '(alien (* rocksdb-mergeoperator)))))
425  ;; concat merge op
426  (with-alien ((state (* t))
427  (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor)))
428  (full-merge (* rocksdb-full-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-full-merge)))
429  (partial-merge (* rocksdb-partial-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-partial-merge)))
430  (delete-value (* rocksdb-delete-value-function) (alien-sap (alien-callable-function 'rocksdb-delete-value)))
431  (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-concat-merge-name))))
432  (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name)
433  '(alien (* rocksdb-mergeoperator)))))
434  (with-opt (o (test-opts) nil)
435  (rocksdb-options-set-merge-operator o (create-dummy-mergeoperator))
436  (with-temp-db db (o)
437  ;; merge_operator=DUMMY in OPTIONS file
438  )))
439 
440 
441 (deftest comparator ()
442  "Test low-level comparator API."
443  (with-alien ((state (* t))
444  (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor)))
445  (compare (* rocksdb-compare-function) (alien-sap (alien-callable-function 'rocksdb-compare-never)))
446  (compare-with-ts (* rocksdb-compare-with-ts-function))
447  (compare-without-ts (* rocksdb-compare-without-ts-function))
448  (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-name))))
449  (is (typep (rocksdb-comparator-create state destructor compare name)
450  '(alien (* rocksdb-comparator))))
451  (is (typep (rocksdb-comparator-with-ts-create state destructor compare compare-with-ts compare-without-ts name)
452  '(alien (* rocksdb-comparator))))
453  ;; TODO - need to test with column-family options
454  (with-opt (o (test-opts) nil)
455  (rocksdb-options-set-comparator
456  o
457  (rocksdb-comparator-create state destructor compare name)))))
458 
459 (deftest compaction ()
460  "Test low-level compactionfilter API."
461  (with-alien ((state (* t))
462  (context (* rocksdb-compactionfiltercontext)))
463  (is (typep
464  (rocksdb-compactionfilter-create state
465  (alien-sap (alien-callable-function 'rocksdb-destructor))
466 (alien-sap (alien-callable-function 'rocksdb-filter-never))
467  (alien-sap (alien-callable-function 'rocksdb-name)))
468  '(alien (* rocksdb-compactionfilter))))
469  (is (typep
470  (rocksdb-compactionfilterfactory-create state
471  (alien-sap (alien-callable-function 'rocksdb-destructor))
472  (alien-sap (alien-callable-function
473  'rocksdb-create-compaction-filter-never))
474  (alien-sap (alien-callable-function 'rocksdb-name)))
475  '(alien (* rocksdb-compactionfilterfactory)))))
476 
477  ;; TODO
478  (with-opt (o (test-opts) nil)
479  (with-temp-db db (o)
480  )))
481 
482 (deftest logger ()
483  "Test logging functionality."
484  (with-alien ((state (* t))
485  (lev unsigned 0)
486  (msg c-string "")
487  (log (* rocksdb-log-function) (alien-sap (alien-callable-function 'rocksdb-log-default))))
488  (is (typep
489  (rocksdb-logger-create-stderr-logger lev msg)
490  '(alien (* rocksdb-logger))))
491  (is (typep
492  (rocksdb-logger-create-callback-logger lev log state)
493  '(alien (* rocksdb-logger))))))
494 
495 (deftest writebatch ()
496  "Test writebatch functionality."
497  (with-alien ((fput (* rocksdb-put-function))
498  (fdeleted (* rocksdb-deleted-function))
499  (fdeleted-cf (* rocksdb-deleted-cf-function))
500  (fput-cf (* rocksdb-put-cf-function))
501  (fmerge-cf (* rocksdb-merge-cf-function))
502  (fget-ts (* rocksdb-get-ts-size-function)))
503  (is (typep
504  (rocksdb-writebatch-create)
505  '(alien (* rocksdb-writebatch))))
506  (is (typep
507  (rocksdb-writebatch-wi-create 0 0)
508  '(alien (* rocksdb-writebatch-wi))))))
509 
510 (deftest slicetransform ()
511  "Test slicetransform functionality."
512  (with-alien ((state (* t))
513  (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor)))
514  (transform (* rocksdb-transform-function))
515  (in-domain (* rocksdb-in-domain-function))
516  (in-range (* rocksdb-in-range-function))
517  (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-name))))
518  (is (typep
519  (rocksdb-slicetransform-create state destructor transform in-domain in-range name)
520  '(alien (* rocksdb-slicetransform))))))