changelog shortlog graph tags branches changeset files revisions annotate raw help

Mercurial > core / lisp/ffi/rocksdb/tests.lisp

changeset 616: c48704d7b06f
parent: 57813b8ee029
child: 5e57683a0c28
author: Richard Westhaver <ellis@rwest.io>
date: Tue, 20 Aug 2024 21:02:58 -0400
permissions: -rw-r--r--
description: writebatch callbacks and fixes
1 ;;; rocksdb/tests.lisp --- RocksDB tests
2 
3 ;;; Code:
4 (defpackage :rocksdb/tests
5  (:use :cl :std :rt :rocksdb :sb-ext :sb-alien :log))
6 
7 (in-package :rocksdb/tests)
8 
9 (defsuite :rocksdb)
10 (in-suite :rocksdb)
11 
12 (load-rocksdb)
13 (init-log-timestamp)
14 
15 (defun rocksdb-test-dir ()
16  (format nil "/tmp/~A/" (gensym "rocksdb-tests-")))
17 
18 (defun rocksdb-test-file ()
19  (format nil "/tmp/~A" (gensym "rocksdb-test-")))
20 
21 (defun test-opts ()
22  (let ((default (rocksdb-options-create)))
23  (rocksdb-options-set-create-if-missing default t)
24  default))
25 
26 ;; not thread safe (gensym-counter)
27 (defun genkey (&optional prefix) (string-to-octets (symbol-name (gensym (or prefix "key")))))
28 (defun genval (&optional prefix) (string-to-octets (symbol-name (gensym (or prefix "val")))))
29 
30 (defmacro with-opt ((var create destroy) &body body)
31  `(let ((,var ,create))
32  (unwind-protect (progn ,@body)
33  ,destroy)))
34 
35 (deftest opts ()
36  (with-opt (o (rocksdb-options-create) (rocksdb-options-destroy o))
37  ;; unsigned-char
38  (rocksdb-options-set-create-if-missing o t)
39  (rocksdb-options-get-create-if-missing o)
40  (rocksdb-options-set-create-missing-column-families o t)
41  (rocksdb-options-get-create-missing-column-families o)
42  (rocksdb-options-set-error-if-exists o t)
43  (rocksdb-options-get-error-if-exists o)
44  (rocksdb-options-set-paranoid-checks o t)
45  (rocksdb-options-get-paranoid-checks o)
46  (rocksdb-options-set-compression-options-use-zstd-dict-trainer o t)
47  (rocksdb-options-get-compression-options-use-zstd-dict-trainer o)
48  (rocksdb-options-set-enable-blob-gc o t)
49  (rocksdb-options-get-enable-blob-gc o)
50  (rocksdb-options-set-allow-ingest-behind o t)
51  (rocksdb-options-get-allow-ingest-behind o)
52  (rocksdb-options-set-skip-stats-update-on-db-open o t)
53  (rocksdb-options-get-skip-stats-update-on-db-open o)
54  (rocksdb-options-set-skip-checking-sst-file-sizes-on-db-open o t)
55  (rocksdb-options-get-skip-checking-sst-file-sizes-on-db-open o)
56  (rocksdb-options-set-enable-blob-files o t)
57  (rocksdb-options-get-enable-blob-files o)
58  (rocksdb-options-set-enable-pipelined-write o t)
59  (rocksdb-options-get-enable-pipelined-write o)
60  (rocksdb-options-set-unordered-write o t)
61  (rocksdb-options-get-unordered-write o)
62  (rocksdb-options-set-allow-mmap-reads o t)
63  (rocksdb-options-get-allow-mmap-reads o)
64  (rocksdb-options-set-allow-mmap-writes o t)
65  (rocksdb-options-get-allow-mmap-writes o)
66  (rocksdb-options-set-use-direct-reads o t)
67  (rocksdb-options-get-use-direct-reads o)
68  (rocksdb-options-set-use-direct-io-for-flush-and-compaction o t)
69  (rocksdb-options-get-use-direct-io-for-flush-and-compaction o)
70  (rocksdb-options-set-is-fd-close-on-exec o t)
71  (rocksdb-options-get-is-fd-close-on-exec o)
72  (rocksdb-options-set-inplace-update-support o t)
73  (rocksdb-options-get-inplace-update-support o)
74  (rocksdb-options-set-advise-random-on-open o t)
75  (rocksdb-options-get-advise-random-on-open o)
76  (rocksdb-options-set-atomic-flush o t)
77  (rocksdb-options-get-atomic-flush o)
78  (rocksdb-options-set-manual-wal-flush o t)
79  (rocksdb-options-get-manual-wal-flush o)
80  (rocksdb-options-set-avoid-unnecessary-blocking-io o t)
81  (rocksdb-options-get-avoid-unnecessary-blocking-io o)
82  ;; this is full-width value 0-255, not boolean
83  (rocksdb-options-set-level-compaction-dynamic-level-bytes o 20)
84  (rocksdb-options-get-level-compaction-dynamic-level-bytes o)
85  ;; int
86  (rocksdb-options-set-compression-options-parallel-threads o 4)
87  (rocksdb-options-get-compression-options-parallel-threads o)
88  (rocksdb-options-set-info-log-level o 1)
89  (rocksdb-options-get-info-log-level o)
90  (rocksdb-options-set-max-open-files o 100)
91  (rocksdb-options-get-max-open-files o)
92  (rocksdb-options-set-max-file-opening-threads o 4)
93  (rocksdb-options-get-max-file-opening-threads o)
94  (rocksdb-options-set-compression-options-zstd-max-train-bytes o 1024)
95  (rocksdb-options-get-compression-options-zstd-max-train-bytes o)
96  (rocksdb-options-set-num-levels o 4)
97  (rocksdb-options-get-num-levels o)
98  (rocksdb-options-set-level0-file-num-compaction-trigger o 16)
99  (rocksdb-options-get-level0-file-num-compaction-trigger o)
100  (rocksdb-options-set-level0-slowdown-writes-trigger o 1024)
101  (rocksdb-options-get-level0-slowdown-writes-trigger o)
102  (rocksdb-options-set-level0-stop-writes-trigger o 1024)
103  (rocksdb-options-get-level0-stop-writes-trigger o)
104  (rocksdb-options-set-target-file-size-multiplier o 4)
105  (rocksdb-options-get-target-file-size-multiplier o)
106  ;; size-t
107  (rocksdb-options-set-write-buffer-size o 1024)
108  (rocksdb-options-get-write-buffer-size o)
109  (rocksdb-options-set-db-write-buffer-size o 1024)
110  (rocksdb-options-get-db-write-buffer-size o)
111  ;; unsigned-long
112  (rocksdb-options-set-compression-options-max-dict-buffer-bytes o 1024)
113  (rocksdb-options-get-compression-options-max-dict-buffer-bytes o)
114  (rocksdb-options-set-max-total-wal-size o 1024)
115  (rocksdb-options-get-max-total-wal-size o)
116  (rocksdb-options-set-target-file-size-base o 1024)
117  (rocksdb-options-get-target-file-size-base o)
118  (rocksdb-options-set-max-bytes-for-level-base o 1024)
119  (rocksdb-options-get-max-bytes-for-level-base o)
120  ;; double
121  ;; (rocksdb-options-set-max-bytes-for-level-multiplier o (the double-float (/ 1 3)))
122  )
123  (let ((opts (rocksdb-options-create))
124  (wopts (rocksdb-writeoptions-create))
125  (ropts (rocksdb-readoptions-create))
126  (bopts (rocksdb-block-based-options-create)))
127  (rocksdb-options-set-create-if-missing opts t)
128  ;; cleanup
129  (rocksdb-options-destroy opts)
130  (rocksdb-writeoptions-destroy wopts)
131  (rocksdb-readoptions-destroy ropts)
132  (rocksdb-block-based-options-destroy bopts)))
133 
134 (defun make-errptr ()
135  (make-alien rocksdb-errptr))
136 
137 (deftest db-basic ()
138  "Test basic RocksDB functionality. Inserts KV pair into a temporary
139 DB where K and V are both Lisp strings."
140  (let* ((opts (test-opts))
141  (path (rocksdb-test-dir))
142  (db (rocksdb-open opts path nil))
143  (key (genkey))
144  (val (genval))
145  (klen (length key))
146  (vlen (length val))
147  (wopts (rocksdb-writeoptions-create))
148  (ropts (rocksdb-readoptions-create)))
149  (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen))
150  (v (* unsigned-char) (make-alien unsigned-char vlen))
151  (errptr rocksdb-errptr nil))
152  ;; copy KEY to K
153  (setfa k key)
154  ;; copy VAL to V
155  (setfa v val)
156  ;; put K:V in DB
157  (rocksdb-put db
158  wopts
159  k
160  klen
161  v
162  vlen
163  errptr)
164  (is (null-alien errptr))
165  ;; get V from DB given K
166  (rocksdb:rocksdb-cancel-all-background-work db t)
167  (rocksdb-get db ropts k klen (make-alien size-t vlen) errptr)
168  (is (null-alien errptr))
169  ;; copy V to RVAL and validate
170  (let ((rval (make-array vlen :element-type 'unsigned-byte)))
171  (loop for i from 0 below vlen do (let ((x (deref v i))) (setf (aref rval i) x)))
172  (is (string= (octets-to-string val) (concatenate 'string (map 'vector #'code-char rval)))))
173  (rocksdb-delete db wopts k klen errptr)
174  (is (null-alien errptr))
175  (rocksdb-writeoptions-destroy wopts)
176  (rocksdb-readoptions-destroy ropts)
177  (rocksdb-cancel-all-background-work db nil)
178  (rocksdb-close db)
179  (rocksdb-destroy-db opts path errptr)
180  (is (null-alien errptr))
181  (rocksdb-options-destroy opts))))
182 
183 (deftest sstfiles ()
184  "Test SST file write/ingest functionality."
185  (let* ((opts (test-opts))
186  (path (rocksdb-test-dir))
187  (file (rocksdb-test-file))
188  (db (rocksdb-open opts path nil))
189  (key (genkey))
190  (val (genval))
191  (klen (length key))
192  (vlen (length val))
193  (eopts (rocksdb-envoptions-create))
194  (iopts (rocksdb-ingestexternalfileoptions-create))
195  (ropts (rocksdb-readoptions-create))
196  (writer (rocksdb-sstfilewriter-create eopts opts)))
197  (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen))
198  (v (* unsigned-char) (make-alien unsigned-char vlen))
199  (flist (array c-string 1))
200  (errptr rocksdb-errptr nil))
201  ;; copy KEY to K
202  (setfa k key)
203  ;; copy VAL to V
204  (setfa v val)
205  (setf (deref flist 0) file)
206  ;; create writer
207  (rocksdb-sstfilewriter-open writer file errptr)
208  ;; insert rows into sst file
209  (rocksdb-sstfilewriter-put writer k klen v vlen errptr)
210  (is (null-alien errptr))
211  (rocksdb-sstfilewriter-finish writer errptr)
212  (is (null-alien errptr))
213  ;; ingest sst file
214  (rocksdb-ingest-external-file db (cast flist (* c-string)) 1 iopts errptr)
215  (is (null-alien errptr))
216  (let ((vres (make-array vlen :element-type 'octet :fill-pointer 0)))
217  (is (string= (octets-to-string val) (cast (rocksdb-get db ropts k klen (make-alien size-t vlen) errptr) c-string))))
218 
219  ;; rocksdb-sstfilewriter-file-size
220  (rocksdb-sstfilewriter-destroy writer)
221  (rocksdb-close db)
222  (rocksdb-destroy-db opts path errptr)
223  (rocksdb-options-destroy opts)
224  (rocksdb-envoptions-destroy eopts)
225  (delete-file file)
226  (is (null-alien errptr)))))
227 
228 (deftest stats ()
229  "Test statistics and performance-context related functionality."
230  (rocksdb-set-perf-level (rocksdb-perf-level "enable-time-except-for-mutex"))
231  (let* ((opts (test-opts))
232  (path (rocksdb-test-dir))
233  (db (rocksdb-open opts path nil))
234  (key (random-bytes 100))
235  (val (random-bytes 100000))
236  (klen (length key))
237  (vlen (length val))
238  (wopts (rocksdb-writeoptions-create))
239  (ropts (rocksdb-readoptions-create))
240  (ctx (rocksdb::rocksdb-perfcontext-create))
241  (hist (rocksdb-statistics-histogram-data-create)))
242  (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen))
243  (v (* (unsigned 8)) (make-alien (unsigned 8) vlen))
244  (errptr rocksdb-errptr nil))
245  ;; copy KEY to K
246  (setfa k key)
247  ;; copy VAL to V
248  (setfa v val)
249  ;; put K:V in DB
250  (rocksdb-put db
251  wopts
252  k
253  klen
254  v
255  vlen
256  errptr)
257 
258  (debug! "stats:" (rocksdb-options-statistics-get-string opts))
259  (rocksdb-options-statistics-get-histogram-data opts 5 hist) ;; histogram data types? uint64 somewhere
260  (debug! "count:" (rocksdb-statistics-histogram-data-get-count hist))
261  (rocksdb-perfcontext-reset ctx)
262  ;; ...
263  (rocksdb-set-perf-level (rocksdb-perf-level "disable"))
264  (rocksdb-statistics-histogram-data-destroy hist)
265  (rocksdb-close db)
266  (rocksdb-destroy-db opts path errptr)
267  (rocksdb-options-destroy opts))))
268 
269 ;; stats-dump-period-sec
270 
271 (deftest blob ()
272  "Test BlobDB functionality."
273  (let* ((opts (test-opts))
274  (path (rocksdb-test-dir))
275  db
276  (key (random-bytes 8))
277  (val (make-array 9999 :initial-element 36))
278  (klen (length key))
279  (vlen (length val))
280  (wopts (rocksdb-writeoptions-create))
281  (ropts (rocksdb-readoptions-create))
282  (bcache (rocksdb-cache-create-lru 128)))
283  (rocksdb-options-set-enable-blob-files opts t)
284  (rocksdb-options-set-enable-blob-gc opts t)
285  (rocksdb-options-set-blob-compression-type opts (rocksdb-compression-backend "zstd"))
286  (rocksdb-options-set-blob-cache opts bcache)
287  (setf db (rocksdb-open opts path nil))
288 
289  (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen))
290  (v (* (unsigned 8)) (make-alien (unsigned 8) vlen))
291  (errptr rocksdb-errptr nil))
292  (debug! "min blob file size: " (rocksdb-options-get-min-blob-size opts))
293  (debug! "max blob file size: " (rocksdb-options-get-blob-file-size opts))
294 
295  ;; copy KEY to K
296  (setfa k key)
297  ;; copy VAL to V
298  (setfa v val)
299  ;; put K:V in DB -
300  (rocksdb-put db
301  wopts
302  k
303  klen
304  v
305  vlen
306  errptr)
307  (is (null-alien errptr))
308  (rocksdb:rocksdb-flush db (rocksdb-flushoptions-create) errptr)
309  (is (null-alien errptr))
310  (is (stringp
311  (cast
312  (rocksdb-get db
313  ropts
314  k
315  klen
316  (make-alien size-t vlen)
317  errptr)
318  c-string)))
319  (rocksdb-writeoptions-destroy wopts)
320  (rocksdb-readoptions-destroy ropts)
321  (rocksdb-close db)
322  (rocksdb-destroy-db opts path errptr)
323  (is (null-alien errptr))
324  (rocksdb-options-destroy opts))))
325 
326 (deftest transaction ()
327  "Test simple transactions using both TransactionDB and OptimisticTransactionDB."
328  (let* ((opts (test-opts))
329  (path (rocksdb-test-dir))
330  (db (rocksdb-open opts path nil))
331  (key (genkey))
332  (val (genval))
333  (klen (length key))
334  (vlen (length val))
335  (wopts (rocksdb-writeoptions-create))
336  (ropts (rocksdb-readoptions-create)))
337  (with-alien ((k (* (unsigned 8)) (make-alien (unsigned 8) klen))
338  (v (* (unsigned 8)) (make-alien (unsigned 8) vlen))
339  (errptr rocksdb-errptr nil))
340  ;; copy KEY to K
341  (setfa k key)
342  ;; copy VAL to V
343  (setfa v val)
344  ;; put K:V in DB -
345  (rocksdb-writeoptions-destroy wopts)
346  (rocksdb-readoptions-destroy ropts)
347  (rocksdb-close db)
348  (rocksdb-destroy-db opts path errptr)
349  (rocksdb-options-destroy opts)
350  (is (null-alien errptr)))))
351 
352 (deftest metadata ()
353  "Test metadata functionality :: cf-meta -> level-meta -> sst-file-meta"
354  nil)
355 
356 (deftest properties ()
357  "Test the ROCKSDB-GET-PROPERTY-* functions."
358  ;; *rocksdb-properties*
359  (let* ((opts (test-opts))
360  (path (rocksdb-test-dir))
361  (db (rocksdb-open opts path nil))
362  (key (genkey))
363  (val (genval))
364  (klen (length key))
365  (vlen (length val))
366  (wopts (rocksdb-writeoptions-create))
367  (ropts (rocksdb-readoptions-create)))
368  (is (stringp (debug! (rocksdb-property-value db (make-alien-string "rocksdb.stats")))))
369  (is (zerop (parse-integer (rocksdb-property-value db (make-alien-string "rocksdb.num-files-at-level3")))))))
370 
371 (deftest merge ()
372  "Test low-level merge-operator functionality using Alien Callbacks."
373  (is (with-alien ((k (array unsigned-char))
374  (v (array unsigned-char))
375  (ops (array (array unsigned-char)))
376  (s (array unsigned-char)))
377  (alien-funcall
378  (alien-callable-function
379  'rocksdb-concat-full-merge)
380  k 0 v 0 ops (make-alien size-t 0) 0 s (make-alien size-t 0))))
381  (is
382  (not
383  (with-alien ((k (array unsigned-char))
384  (ops (array (array unsigned-char)))
385  (s (array unsigned-char)))
386  (alien-funcall
387  (alien-callable-function
388  'rocksdb-concat-partial-merge)
389  k 0 ops (make-alien size-t 0) 0 s (make-alien size-t 0)))))
390  (alien-callable-function 'rocksdb-concat-full-merge)
391  (alien-callable-function 'rocksdb-concat-partial-merge)
392  (is (integerp
393  (parse-integer
394  (string-trim "rocksdb:" (alien-funcall (alien-callable-function 'rocksdb-name))))))
395  ;; returns No Value
396  (with-alien ((str c-string (make-alien-string ""))
397  (state (* t)))
398  (is (null (alien-funcall (alien-callable-function 'rocksdb-delete-value) state str 1))))
399 
400  (is (null (alien-funcall (alien-callable-function 'rocksdb-destructor) (make-alien (* t)))))
401 
402  ;; null merge op
403  (with-alien ((state (* t))
404  (destructor (* rocksdb-destructor-function))
405  (full-merge (* rocksdb-full-merge-function))
406  (partial-merge (* rocksdb-partial-merge-function))
407  (delete-value (* rocksdb-delete-value-function))
408  (name (* rocksdb-name-function)))
409  (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name)
410  '(alien (* rocksdb-mergeoperator)))))
411 
412  ;; concat merge op
413  (with-alien ((state (* t))
414  (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor)))
415  (full-merge (* rocksdb-full-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-full-merge)))
416  (partial-merge (* rocksdb-partial-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-partial-merge)))
417  (delete-value (* rocksdb-delete-value-function) (alien-sap (alien-callable-function 'rocksdb-delete-value)))
418  (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-concat-merge-name))))
419  (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name)
420  '(alien (* rocksdb-mergeoperator))))))
421 
422 (deftest comparator ()
423  "Test low-level comparator API."
424  (with-alien ((state (* t))
425  (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor)))
426  (compare (* rocksdb-compare-function) (alien-sap (alien-callable-function 'rocksdb-compare-never)))
427  (compare-with-ts (* rocksdb-compare-with-ts-function))
428  (compare-without-ts (* rocksdb-compare-without-ts-function))
429  (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-name))))
430  (is (typep (rocksdb-comparator-create state destructor compare name)
431  '(alien (* rocksdb-comparator))))
432  (is (typep (rocksdb-comparator-with-ts-create state destructor compare compare-with-ts compare-without-ts name)
433  '(alien (* rocksdb-comparator))))))
434 
435 (deftest compaction ()
436  "Test low-level compactionfilter API."
437  (with-alien ((state (* t))
438  (context (* rocksdb-compactionfiltercontext)))
439  (is (typep
440  (rocksdb-compactionfilter-create state
441  (alien-sap (alien-callable-function 'rocksdb-destructor))
442 (alien-sap (alien-callable-function 'rocksdb-filter-never))
443  (alien-sap (alien-callable-function 'rocksdb-name)))
444  '(alien (* rocksdb-compactionfilter))))
445  (is (typep
446  (rocksdb-compactionfilterfactory-create state
447  (alien-sap (alien-callable-function 'rocksdb-destructor))
448  (alien-sap (alien-callable-function
449  'rocksdb-create-compaction-filter-never))
450  (alien-sap (alien-callable-function 'rocksdb-name)))
451  '(alien (* rocksdb-compactionfilterfactory))))))
452 
453 (deftest logger ()
454  "Test logging functionality."
455  (with-alien ((state (* t))
456  (lev unsigned 0)
457  (msg c-string)
458  (log (* rocksdb-log-function) (alien-sap (alien-callable-function 'rocksdb-log-default))))
459  (is (typep
460  (rocksdb-logger-create-stderr-logger lev msg)
461  '(alien (* rocksdb-logger))))
462  (is (typep
463  (rocksdb-logger-create-callback-logger lev log state)
464  '(alien (* rocksdb-logger))))))
465 
466 (deftest writebatch ()
467  "Test writebatch functionality."
468  (with-alien ((fput (* rocksdb-put-function))
469  (fdeleted (* rocksdb-deleted-function))
470  (fdeleted-cf (* rocksdb-deleted-cf-function))
471  (fput-cf (* rocksdb-put-cf-function))
472  (fmerge-cf (* rocksdb-merge-cf-function))
473  (fget-ts (* rocksdb-get-ts-size-function)))
474  (is (typep
475  (rocksdb-writebatch-create)
476  '(alien (* rocksdb-writebatch))))
477  (is (typep
478  (rocksdb-writebatch-wi-create 0 0)
479  '(alien (* rocksdb-writebatch-wi))))))
480 
481 (deftest slicetransform (:skip t)
482  "Test slicetransform functionality."
483  (with-alien ((state (* t))
484  (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor)))
485  (transform (* t) (* rocksdb-transform-function))
486  (in-domain (* rocksdb-in-domain-function))
487  (in-range (* rocksdb-in-range-function))
488  (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-name))))
489  (rocksdb-slicetransform-create state destructor transform in-domain in-range name)))