changeset 698: | 96958d3eb5b0 |
parent: | 2e7d93b892a5 |
author: | Richard Westhaver <ellis@rwest.io> |
date: | Fri, 04 Oct 2024 22:04:59 -0400 |
permissions: | -rw-r--r-- |
description: | fixes |
226 | 1 | ;;; rdb/macs.lisp --- macros |
2 | ||
3 | ;;; Code: |
|
93 | 4 | (in-package :rdb) |
5 | ||
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
6 | ;;; error handling |
226 | 7 | (defmacro with-errptr ((e &optional errtyp params) &body body) |
8 | `(with-alien ((,e rocksdb-errptr nil)) |
|
9 | (unwind-protect |
|
309
94d358919982
refactor rust, fixing cli issues and rdb error handling
Richard Westhaver <ellis@rwest.io>
parents:
293
diff
changeset
|
10 | (handler-bind ((sb-sys:memory-fault-error |
94d358919982
refactor rust, fixing cli issues and rdb error handling
Richard Westhaver <ellis@rwest.io>
parents:
293
diff
changeset
|
11 | (lambda (c) |
680
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
679
diff
changeset
|
12 | (declare (ignore c)) |
309
94d358919982
refactor rust, fixing cli issues and rdb error handling
Richard Westhaver <ellis@rwest.io>
parents:
293
diff
changeset
|
13 | (handle-errptr ,e ,errtyp ,params))) |
94d358919982
refactor rust, fixing cli issues and rdb error handling
Richard Westhaver <ellis@rwest.io>
parents:
293
diff
changeset
|
14 | (error |
94d358919982
refactor rust, fixing cli issues and rdb error handling
Richard Westhaver <ellis@rwest.io>
parents:
293
diff
changeset
|
15 | (lambda (c) |
680
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
679
diff
changeset
|
16 | (declare (ignore c)) |
309
94d358919982
refactor rust, fixing cli issues and rdb error handling
Richard Westhaver <ellis@rwest.io>
parents:
293
diff
changeset
|
17 | (handle-errptr ,e ,errtyp ,params)))) |
226 | 18 | (progn ,@body)) |
679
12287fab15d0
rocksdb load opts and env updates
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
19 | (handle-errptr ,e ,errtyp ,params)))) |
226 | 20 | |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
21 | ;;; opts |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
22 | (defmacro rdb-opt-setter (key) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
23 | `(find-symbol (format nil "~:@(rocksdb-options-set-~x~)" ,key) :rocksdb)) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
24 | |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
25 | (defmacro rdb-opt-getter (key) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
26 | `(find-symbol (format nil "~:@(rocksdb-options-get-~x~)" ,key) :rocksdb)) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
27 | |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
28 | ;;; db |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
29 | (defmacro with-open-db-raw ((db-var db-path &optional (opt (default-rocksdb-options))) &body body) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
30 | `(let ((,db-var (open-db-raw ,db-path ,opt))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
31 | (unwind-protect (progn ,@body) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
32 | (rocksdb-close ,db-var) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
33 | (with-errptr (err 'rocksdb-error) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
34 | ;; (rocksdb-destroy-db ,opt ,db-path err) ;; when :destroy only |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
35 | (rocksdb-options-destroy ,opt))))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
36 | |
689 | 37 | (defmacro with-db ((db-var db &key open close) &body body) |
269 | 38 | "Bind DB-VAR to the database object DB for the lifetime of BODY." |
112 | 39 | `(let ((,db-var ,db)) |
40 | (handler-bind ((error (lambda (condition) |
|
41 | (error 'rdb-error |
|
42 | :message |
|
269 | 43 | (format nil "WITH-DB signaled: ~A" condition))))) |
689 | 44 | ,@(when open `(open-db ,db-var)) |
45 | ,@(if close `(unwind-protect (progn ,@body) (close ,db-var)) |
|
46 | body)))) |
|
93 | 47 | |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
48 | ;;; cf |
93 | 49 | (defmacro with-cf ((cf-var cf) &body body) |
269 | 50 | "Bind CF to CF-VAR for the lifetime of BODY." |
93 | 51 | `(let ((,cf-var ,cf)) |
269 | 52 | (handler-bind ((error (lambda (condition) |
53 | (error 'cf-error |
|
54 | :message |
|
55 | (format nil "WITH-CF signaled: ~A" condition))))) |
|
56 | ,@body))) |
|
267
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
57 | |
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
58 | (defmacro do-cfs ((cf cfs) &body body) |
269 | 59 | "Do BODY for each CF in the array CFS." |
267
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
60 | (with-gensyms (%cf) |
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
61 | `(loop for ,%cf across ,cfs |
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
62 | do (with-cf (,cf ,%cf) ,@body)))) |
269 | 63 | |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
64 | ;;; iter |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
65 | (defmacro with-iter-raw ((iter-var db &optional (opt (rocksdb-readoptions-create))) &body body) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
66 | `(let ((,iter-var (create-iter-raw ,db ,opt))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
67 | (unwind-protect (progn ,@body) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
68 | (destroy-iter-raw ,iter-var)))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
69 | |
269 | 70 | (defmacro with-iter ((iter-var iter) &body body) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
71 | "Bind object ITER to ITER-VAR. |
269 | 72 | |
73 | ((%ITER ITER) BODY) is passed to ROCKSDB:WITH-ITER-RAW, binding the |
|
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
74 | raw handle to the same symbol prefixed with '%'. |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
75 | |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
76 | Errors that occur in the inner body will be handled but the iterator |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
77 | handle will not be freed on exit." |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
78 | (let ((%iter-var (symbolicate '% (symbol-name iter-var)))) |
269 | 79 | `(let ((,iter-var ,iter)) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
80 | (let ((,%iter-var (rdb-iter-sap ,iter-var))) |
310 | 81 | (declare (ignorable ,%iter-var)) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
82 | ,@body)))) |
269 | 83 | |
84 | ;; TODO: sb-ext:with-current-source-form ? |
|
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
85 | ;;; backup |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
86 | (defmacro with-open-backup-engine-raw ((be-var be-path &optional (opt (rocksdb-options-create))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
87 | &body body) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
88 | `(let ((,be-var (open-backup-engine-raw ,be-path ,opt))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
89 | (unwind-protect (progn ,@body) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
90 | (rocksdb-backup-engine-close ,be-var)))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
91 | |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
92 | ;;; top-level |
679
12287fab15d0
rocksdb load opts and env updates
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
93 | ;; TODO 2024-09-26: |
269 | 94 | (defmacro do-db ((db opts) accessors &body body) |
95 | "Database Iteration construct. OPTS are used to provide top-level |
|
96 | options dynamically bound to DB. ACCESSORS is a list of database |
|
679
12287fab15d0
rocksdb load opts and env updates
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
97 | accessors which are available to call in BODY.") |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
98 | |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
99 | ;;; temp-db |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
100 | (defvar *temp-db-path-generator* |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
101 | (lambda (&optional (name "temp-db")) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
102 | (make-pathname :directory "tmp" :name (symbol-name (gensym name)))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
103 | "A single arg function returning the absolute path to a temp-db path.") |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
104 | |
274
5f782d361e08
threads and db tweaks. fixed a tricky macro error caused by string-case, dat/html now works.
Richard Westhaver <ellis@rwest.io>
parents:
273
diff
changeset
|
105 | (defvar *temp-db-destroy* nil) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
106 | |
274
5f782d361e08
threads and db tweaks. fixed a tricky macro error caused by string-case, dat/html now works.
Richard Westhaver <ellis@rwest.io>
parents:
273
diff
changeset
|
107 | (defmacro with-temp-db ((db-var (&rest cfs) &key (destroy *temp-db-destroy*) open) &body body) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
108 | "Bind DB-VAR to a temporary RDB object, arranging for CF-VARS to be |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
109 | created as column-families and destroying the database after executing |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
110 | the forms in BODY." |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
111 | (setf cfs |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
112 | (mapcar |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
113 | (lambda (var) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
114 | (setf var (make-rdb-cf (symbol-name var)))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
115 | cfs)) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
116 | `(with-db (,db-var (make-rdb |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
117 | (namestring (funcall ,*temp-db-path-generator* ,(symbol-name db-var))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
118 | (default-rdb-opts) |
624
97dd03beda03
rocksdb updates in coordination with infra/scripts/org-graph-db-init.lisp
Richard Westhaver <ellis@rwest.io>
parents:
310
diff
changeset
|
119 | (make-array ,(length cfs) :element-type 'rdb-cf :initial-contents ',cfs :adjustable t :fill-pointer ,(length cfs)))) |
274
5f782d361e08
threads and db tweaks. fixed a tricky macro error caused by string-case, dat/html now works.
Richard Westhaver <ellis@rwest.io>
parents:
273
diff
changeset
|
120 | ,@(when open `((open-db ,db-var) |
5f782d361e08
threads and db tweaks. fixed a tricky macro error caused by string-case, dat/html now works.
Richard Westhaver <ellis@rwest.io>
parents:
273
diff
changeset
|
121 | (create-cfs ,db-var))) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
122 | (prog1 |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
123 | (progn ,@body) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
124 | ,(if destroy |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
125 | `(destroy-db ,db-var) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
126 | `(shutdown-db ,db-var))))) |
293
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
127 | ;;; sst |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
128 | (defmacro with-sst ((sst &key file comparator destroy) &body body) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
129 | "Do BODY with SST bound to a SST-FILE-WRITER. When FILE is supplied |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
130 | the writer will automatically open that file. |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
131 | |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
132 | When COMPARATOR is supplied it is used as the comparator function for |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
133 | the writer. Every key inserted MUST be in ascending order, according |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
134 | to the comparator. By default the ordering is binary |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
135 | lexicographically. |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
136 | |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
137 | It is up to the developer to ensure that the comparator used by a |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
138 | writer is exactly the same as the comparator used when ingesting the |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
139 | file by a RDB instance." |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
140 | `(let ((,sst (make-sst-file-writer ,comparator))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
141 | ,@(when file `((open-sst ,sst ,file))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
142 | ,@body |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
274
diff
changeset
|
143 | ,@(when destroy `((destroy-sst ,sst))))) |
679
12287fab15d0
rocksdb load opts and env updates
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
144 | |
12287fab15d0
rocksdb load opts and env updates
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
145 | ;;; opts |
680
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
679
diff
changeset
|
146 | (defmacro with-latest-opts (db &body body) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
679
diff
changeset
|
147 | `(progn |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
679
diff
changeset
|
148 | (let ((,db (load-opts ,db))) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
679
diff
changeset
|
149 | ,@body))) |