changeset 698: | 96958d3eb5b0 |
parent: | 57813b8ee029 |
author: | Richard Westhaver <ellis@rwest.io> |
date: | Fri, 04 Oct 2024 22:04:59 -0400 |
permissions: | -rw-r--r-- |
description: | fixes |
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
1 | ;;; rocksdb/compaction.lisp --- RocksDB Compaction |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
2 | |
595 | 3 | ;; RocksDB Lisp Compaction Filter API |
4 | ||
5 | ;;; Commentary: |
|
6 | ||
7 | ;; compaction filters are like custom GC rules for the database. compactions |
|
8 | ;; run in the background and can be configured via the column-family-options |
|
9 | ;; or compactionfilterfactory API. |
|
10 | ||
11 | ;; ref: https://github.com/facebook/rocksdb/wiki/Compaction-Filter |
|
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
12 | |
606
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
13 | #| |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
14 | * RocksDB snapshots do not guarantee to preserve the state of the DB in the |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
15 | presence of CompactionFilter. Data seen from a snapshot might disappear after |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
16 | a table file created with a `CompactionFilter` is installed. If you use |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
17 | snapshots, think twice about whether you want to use `CompactionFilter` and |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
18 | whether you are using it in a safe way. |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
19 | |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
20 | * If multithreaded compaction is being used *and* a single CompactionFilter |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
21 | instance was supplied via Options::compaction_filter, CompactionFilter |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
22 | methods may be called from different threads concurrently. The application |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
23 | must ensure that such calls are thread-safe. If the CompactionFilter was |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
24 | created by a factory, then it will only ever be used by a single thread that |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
25 | is doing the table file creation, and this call does not need to be |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
26 | thread-safe. However, multiple filters may be in existence and operating |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
27 | concurrently. |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
28 | |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
29 | * The key passed to the filtering methods includes the timestamp if |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
30 | user-defined timestamps are enabled. |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
31 | |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
32 | * Exceptions MUST NOT propagate out of overridden functions into RocksDB, |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
33 | because RocksDB is not exception-safe. This could cause undefined behavior |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
34 | including data loss, unreported corruption, deadlocks, and more. |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
35 | |# |
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
36 | ;;; Code: |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
37 | (in-package :rocksdb) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
38 | |
598
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
39 | (define-alien-type rocksdb-filter-function |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
40 | (function unsigned-char |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
41 | (* t) |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
42 | int |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
43 | c-string |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
44 | size-t |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
45 | c-string |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
46 | size-t |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
47 | (* (array unsigned-char)) |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
48 | (* size-t) |
c7f9bfc9570f
rm duplicate files, add rocksdb batch and logger files
Richard Westhaver <ellis@rwest.io>
parents:
595
diff
changeset
|
49 | (* unsigned-char))) |
603
6e5be24bf789
fix compaction type ordering
Richard Westhaver <ellis@rwest.io>
parents:
599
diff
changeset
|
50 | |
6e5be24bf789
fix compaction type ordering
Richard Westhaver <ellis@rwest.io>
parents:
599
diff
changeset
|
51 | (define-alien-type rocksdb-create-compaction-filter-function |
6e5be24bf789
fix compaction type ordering
Richard Westhaver <ellis@rwest.io>
parents:
599
diff
changeset
|
52 | (function (* rocksdb-compactionfilter) |
6e5be24bf789
fix compaction type ordering
Richard Westhaver <ellis@rwest.io>
parents:
599
diff
changeset
|
53 | (* t) |
6e5be24bf789
fix compaction type ordering
Richard Westhaver <ellis@rwest.io>
parents:
599
diff
changeset
|
54 | (* rocksdb-compactionfiltercontext))) |
606
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
55 | |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
56 | (define-alien-routine rocksdb-compactionfilter-create (* rocksdb-compactionfilter) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
57 | (state (* t)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
58 | (destructor (* rocksdb-destructor-function)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
59 | (filter (* rocksdb-filter-function)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
60 | (name (* rocksdb-name-function))) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
61 | |
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
62 | (define-alien-routine rocksdb-compactionfilter-set-ignore-snapshots void |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
63 | (self (* rocksdb-compactionfilter)) (val unsigned-char)) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
64 | |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
65 | (define-alien-routine rocksdb-compactionfilter-destroy void |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
66 | (self (* rocksdb-compactionfilter))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
67 | |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
68 | ;;; Compaction Filter Context |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
69 | (define-alien-routine rocksdb-compactionfiltercontext-is-full-compaction unsigned-char |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
70 | (context (* rocksdb-compactionfiltercontext))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
71 | |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
72 | (define-alien-routine rocksdb-compactionfiltercontext-is-manual-compaction unsigned-char |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
73 | (context (* rocksdb-compactionfiltercontext))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
74 | |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
diff
changeset
|
75 | ;;; Compaction Filter Factory |
606
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
76 | (define-alien-routine rocksdb-compactionfilterfactory-create (* rocksdb-compactionfilterfactory) |
594
5bd0eb9fa1fa
rocksdb callbacks, missing symbol fixes
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
77 | (state (* t)) |
599 | 78 | (destructor (* rocksdb-destructor-function)) |
606
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
79 | (creator (* rocksdb-create-compaction-filter-function)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
80 | (name (* rocksdb-name-function))) |
594
5bd0eb9fa1fa
rocksdb callbacks, missing symbol fixes
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
81 | |
606
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
82 | (define-alien-routine rocksdb-compacitonfilterfactory-destroy void |
594
5bd0eb9fa1fa
rocksdb callbacks, missing symbol fixes
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
83 | (factory (* rocksdb-compactionfilterfactory))) |
5bd0eb9fa1fa
rocksdb callbacks, missing symbol fixes
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
84 | |
606
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
85 | (define-alien-callable rocksdb-filter-never unsigned-char |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
86 | ((state (* t)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
87 | (level int) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
88 | (key (array unsigned-char)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
89 | (key-length size-t) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
90 | (existing-val (array unsigned-char)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
91 | (existing-val-length size-t) |
611
57813b8ee029
more rocksdb work, slice -> slicetransform
Richard Westhaver <ellis@rwest.io>
parents:
606
diff
changeset
|
92 | (new-val (* (array unsigned-char))) |
57813b8ee029
more rocksdb work, slice -> slicetransform
Richard Westhaver <ellis@rwest.io>
parents:
606
diff
changeset
|
93 | (new-val-length (* size-t)) |
606
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
94 | (value-changed (* unsigned-char))) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
95 | (declare (ignore state level key key-length existing-val existing-val-length new-val new-val-length value-changed)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
96 | 0) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
97 | |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
98 | (define-alien-callable rocksdb-create-compaction-filter-never (* rocksdb-compactionfilter) |
594
5bd0eb9fa1fa
rocksdb callbacks, missing symbol fixes
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
99 | ((state (* t)) |
5bd0eb9fa1fa
rocksdb callbacks, missing symbol fixes
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
100 | (context (* rocksdb-compactionfiltercontext))) |
606
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
101 | (rocksdb-compactionfilter-create |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
102 | state |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
103 | (alien-sap (alien-callable-function 'rocksdb-destructor)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
104 | (alien-sap (alien-callable-function 'rocksdb-filter-never)) |
6fc04c4d465c
emacs and rocksdb upgrades
Richard Westhaver <ellis@rwest.io>
parents:
603
diff
changeset
|
105 | context)) |