changeset 680: | 5f88b237ce29 |
parent: | 97dd03beda03 |
author: | Richard Westhaver <ellis@rwest.io> |
date: | Fri, 27 Sep 2024 20:19:10 -0400 |
permissions: | -rw-r--r-- |
description: | added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb |
226 | 1 | ;;; rdb/raw.lisp --- Raw wrappers for ROCKSDB alien interface |
2 | ||
3 | ;;; Code: |
|
93 | 4 | (in-package :rdb) |
5 | ||
271
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
6 | ;;; Options |
99 | 7 | (defun make-rocksdb-options (&optional init-fn) |
8 | "Make and return RDB-OPTS. INIT-FN is an optional argument which must be a |
|
9 | lambda which takes a single parameter (the RDB-OPTS sap). It is used |
|
10 | to initialize the instance with custom configuration." |
|
11 | (let ((opts (rocksdb-options-create))) |
|
12 | (when init-fn (funcall init-fn opts)) |
|
13 | opts)) |
|
93 | 14 | |
15 | (defun default-rocksdb-options () |
|
99 | 16 | (make-rocksdb-options |
282 | 17 | (lambda (o) (rocksdb-options-set-create-if-missing o t)))) |
93 | 18 | |
680
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
19 | (defun load-opts-raw (dir) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
20 | (rocksdb::with-latest-options dir (db-opts names cf-opts) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
21 | (values db-opts names cf-opts))) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
22 | |
271
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
23 | (defun get-stats-raw (opt htype) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
24 | (with-alien ((hist (* rocksdb-statistics-histogram-data) (rocksdb-statistics-histogram-data-create))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
25 | (rocksdb-options-statistics-get-histogram-data opt htype hist) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
26 | (deref hist))) |
271
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
27 | |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
28 | ;;; DB |
110
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
29 | (defun open-db-raw (db-path &optional (opts (default-rocksdb-options))) |
102 | 30 | (with-errptr (err 'open-db-error (list :db db-path)) |
93 | 31 | (let* ((db-path (if (pathnamep db-path) |
32 | (namestring db-path) |
|
102 | 33 | db-path))) |
34 | (rocksdb-open opts db-path err)))) |
|
93 | 35 | |
36 | (defun close-db-raw (db) |
|
37 | (rocksdb-close db)) |
|
38 | ||
111 | 39 | (defun destroy-db-raw (path &optional (opt (rocksdb-options-create))) |
102 | 40 | (with-errptr (err 'destroy-db-error (list :db path)) |
267
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
41 | (rocksdb-destroy-db opt (namestring (uiop:ensure-directory-pathname path)) err) |
111 | 42 | (rocksdb-options-destroy opt))) |
99 | 43 | |
271
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
44 | (defun get-property-raw (db str) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
45 | (rocksdb-property-value db (make-alien-string str))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
46 | |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
47 | (defun get-metadata-raw (db &optional cf) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
48 | (if cf |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
49 | (rocksdb-get-column-family-metadata-cf db cf) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
50 | (rocksdb-get-column-family-metadata db))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
51 | |
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
52 | (defun flush-db-raw (db &optional (opts (rocksdb-flushoptions-create))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
53 | (with-errptr (err 'flush-db-error (list :db db)) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
54 | (rocksdb-flush db opts err))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
55 | |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
56 | (defun repair-db-raw (name &optional (opts (rocksdb-options-create))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
57 | (with-errptr (err 'repair-db-error (list :name name)) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
58 | (rocksdb-repair-db opts name err))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
59 | |
292
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
60 | (defun ingest-db-raw (db files &optional (opts (rocksdb-ingestexternalfileoptions-create))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
61 | (let ((flen (length files))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
62 | (with-errptr (err 'ingest-db-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
63 | (with-alien ((flist (* c-string) (make-alien c-string flen))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
64 | (loop for f in files |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
65 | for i from 0 to flen |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
66 | do (setf (deref flist i) (make-alien-string f :null-terminate nil))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
67 | (rocksdb-ingest-external-file db flist flen opts err))))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
68 | |
293
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
69 | (defun ingest-db-cf-raw (db cf files &optional (opts (rocksdb-ingestexternalfileoptions-create))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
70 | (let ((flen (length files))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
71 | (with-errptr (err 'ingest-db-error) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
72 | (with-alien ((flist (* c-string) (make-alien c-string flen))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
73 | (loop for f in files |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
74 | for i from 0 to flen |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
75 | do (setf (deref flist i) (make-alien-string f :null-terminate nil))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
76 | (rocksdb-ingest-external-file-cf db cf flist flen opts err))))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
77 | |
271
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
78 | ;;; KVs |
93 | 79 | (defun put-kv-raw (db key val &optional (opts (rocksdb-writeoptions-create))) |
80 | (let ((klen (length key)) |
|
81 | (vlen (length val))) |
|
267
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
82 | (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen)) |
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
83 | (v (* unsigned-char) (make-alien unsigned-char vlen))) |
93 | 84 | (setfa k key) |
85 | (setfa v val) |
|
102 | 86 | (with-errptr (err 'put-kv-error (list :db db :kv (cons key val))) |
98 | 87 | (rocksdb-put db |
88 | opts |
|
89 | k |
|
90 | klen |
|
91 | v |
|
92 | vlen |
|
102 | 93 | err))))) |
93 | 94 | |
110
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
95 | (defun put-kv-str-raw (db key val &optional (opts (rocksdb-writeoptions-create))) |
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
96 | (let ((key-octets (string-to-octets key :null-terminate nil)) |
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
97 | (val-octets (string-to-octets val :null-terminate nil))) |
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
98 | (put-kv-raw db key-octets val-octets opts))) |
93 | 99 | |
100 | (defun get-kv-raw (db key &optional (opt (rocksdb-readoptions-create))) |
|
101 | (let ((klen (length key))) |
|
102 | 102 | (with-errptr (err 'get-kv-error (list :db db :key key)) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
103 | (with-alien ((vlen size-t) |
267
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
104 | (k (* unsigned-char) (make-alien unsigned-char klen))) |
99 | 105 | (setfa k key) |
106 | (let* ((val (rocksdb-get db |
|
107 | opt |
|
102 | 108 | k klen |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
109 | (addr vlen) |
99 | 110 | err))) |
111 | ;; helps if we know the vlen beforehand, would need a custom |
|
112 | ;; C-side function probably. |
|
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
113 | (let ((v (make-array vlen :element-type 'octet))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
114 | (clone-octets-from-alien val v vlen) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
115 | (coerce v 'octet-vector))))))) |
93 | 116 | |
110
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
117 | (defun get-kv-str-raw (db key &optional (opt (rocksdb-readoptions-create))) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
118 | (let ((k (string-to-octets key))) |
99 | 119 | (let ((v (get-kv-raw db k opt))) |
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
120 | (when v (octets-to-string v))))) |
93 | 121 | |
271
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
122 | ;;; Column Family |
680
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
123 | (defun open-cfs-raw (db-opt name names opts) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
124 | (let ((n (length names))) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
125 | (with-alien ((cf-names (* c-string) (clone-strings names)) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
126 | (cf-opts (* (* rocksdb-options))) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
127 | (cf-handles (* (* rocksdb-column-family-handle)))) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
128 | (loop for opt in opts |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
129 | for i below n |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
130 | do (setf (deref cf-opts i) opt)) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
131 | (with-errptr (err 'rocksdb-cf-error (list :cf name)) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
132 | (let ((db (rocksdb-open-column-families db-opt name n cf-names cf-opts cf-handles err))) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
133 | (values db cf-handles)))))) |
624
97dd03beda03
rocksdb updates in coordination with infra/scripts/org-graph-db-init.lisp
Richard Westhaver <ellis@rwest.io>
parents:
293
diff
changeset
|
134 | |
271
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
135 | (defun create-cf-raw (db name &optional (opt (rocksdb-options-create))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
136 | (with-errptr (err 'rocksdb-cf-error (list :db db :cf name)) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
137 | (rocksdb-create-column-family db opt name err))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
138 | |
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
139 | (defun destroy-cf-raw (cf) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
140 | (rocksdb-column-family-handle-destroy cf)) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
141 | |
93 | 142 | (defun get-cf-raw (db cf key &optional (opt (rocksdb-readoptions-create))) |
143 | (let ((klen (length key))) |
|
102 | 144 | (with-errptr (err 'get-kv-error (list :db db :key key)) |
99 | 145 | (with-alien ((vlen (* size-t) (make-alien size-t 0)) |
267
f3d814fb136a
db upgrades, alik, ignoring c files (for now)
Richard Westhaver <ellis@rwest.io>
parents:
226
diff
changeset
|
146 | (k (* unsigned-char) (make-alien unsigned-char klen))) |
99 | 147 | (setfa k key) |
148 | (let* ((val (rocksdb-get-cf db |
|
149 | opt |
|
150 | cf |
|
102 | 151 | k klen |
99 | 152 | vlen |
153 | err))) |
|
154 | ;; helps if we know the vlen beforehand, would need a custom |
|
155 | ;; C-side function probably. |
|
156 | (let ((v (make-array (deref vlen) :element-type 'unsigned-byte))) |
|
157 | (clone-octets-from-alien val v (deref vlen)) |
|
158 | v)))))) |
|
93 | 159 | |
110
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
160 | (defun get-cf-str-raw (db cf key &optional (opt (rocksdb-readoptions-create))) |
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
161 | (let ((k (string-to-octets key :null-terminate nil))) |
99 | 162 | (let ((v (get-cf-raw db cf k opt))) |
163 | (when v (concatenate 'string (map 'vector #'code-char v)))))) |
|
93 | 164 | |
271
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
165 | (defun put-cf-raw (db cf key val &optional (opts (rocksdb-writeoptions-create))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
166 | (let ((klen (length key)) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
167 | (vlen (length val))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
168 | (with-errptr (err 'put-kv-error (list :db db :kv (cons key val))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
169 | (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen)) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
170 | (v (* unsigned-char) (make-alien unsigned-char vlen))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
171 | (setfa k key) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
172 | (setfa v val) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
173 | (rocksdb-put-cf db |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
174 | opts |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
175 | cf |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
176 | k klen |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
177 | v vlen |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
178 | err))))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
179 | |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
180 | (defun put-cf-str-raw (db cf key val &optional (opt (rocksdb-writeoptions-create))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
181 | (let ((key-octets (string-to-octets key :null-terminate nil)) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
182 | (val-octets (string-to-octets val :null-terminate nil))) |
4a1a21ff46ee
metadata,props,merge-ops
Richard Westhaver <ellis@rwest.io>
parents:
267
diff
changeset
|
183 | (put-cf-raw db cf key-octets val-octets opt))) |
110
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
184 | |
680
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
185 | (defun cf-name-raw (cf-handle) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
186 | (rocksdb-column-family-handle-get-name cf-handle (make-alien unsigned-long))) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
187 | |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
188 | (defun cf-id-raw (cf-handle) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
189 | (rocksdb-column-family-handle-get-id cf-handle)) |
5f88b237ce29
added skc, fixed alien c-string functions, upgrades and fixes for rocksdb/rdb
Richard Westhaver <ellis@rwest.io>
parents:
624
diff
changeset
|
190 | |
110
cae8da4b1415
rocksdb macrofication, fixes for RDB (C Strings will be the death of me), removed PWN - quicklisp package conflict, smh.
ellis <ellis@rwest.io>
parents:
102
diff
changeset
|
191 | ;;; Iterators |
111 | 192 | (defun create-iter-raw (db &optional (opt (rocksdb-readoptions-create))) |
93 | 193 | (rocksdb-create-iterator db opt)) |
194 | ||
273
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
195 | (defun create-cf-iter-raw (db cf &optional (opt (rocksdb-readoptions-create))) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
196 | (rocksdb-create-iterator-cf db opt cf)) |
6d56c4950fa2
rocksdb iter,metadata,and more
Richard Westhaver <ellis@rwest.io>
parents:
272
diff
changeset
|
197 | |
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
198 | (defun destroy-iter-raw (iter) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
199 | (rocksdb-iter-destroy iter)) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
200 | |
111 | 201 | (defun iter-key-raw (iter) |
93 | 202 | (with-alien ((klen-ptr (* size-t) (make-alien size-t 0))) |
203 | (let* ((key-ptr (rocksdb-iter-key iter klen-ptr)) |
|
204 | (klen (deref klen-ptr)) |
|
205 | (k (make-array klen :element-type '(unsigned-byte 8)))) |
|
206 | (clone-octets-from-alien key-ptr k klen) |
|
207 | k))) |
|
208 | ||
111 | 209 | (defun iter-key-str-raw (iter) |
210 | (when-let ((k (iter-key-raw iter))) |
|
93 | 211 | (octets-to-string k))) |
212 | ||
111 | 213 | (defun iter-val-raw (iter) |
99 | 214 | (with-alien ((vlen-ptr (* size-t) (make-alien size-t 0))) |
215 | (let* ((val-ptr (rocksdb-iter-value iter vlen-ptr)) |
|
216 | (vlen (deref vlen-ptr)) |
|
217 | (v (make-array vlen :element-type '(unsigned-byte 8)))) |
|
218 | (clone-octets-from-alien val-ptr v vlen) |
|
219 | v))) |
|
93 | 220 | |
111 | 221 | (defun iter-val-str-raw (iter) |
222 | (when-let ((v (iter-val-raw iter))) |
|
99 | 223 | (octets-to-string v))) |
111 | 224 | |
112 | 225 | ;;; Backup Engine |
226 | (defun open-backup-engine-raw (be-path &optional (opts (rocksdb-options-create))) |
|
227 | (with-errptr (err 'open-backup-engine-error (list :db be-path)) |
|
228 | (let ((be-path (if (pathnamep be-path) |
|
229 | (namestring be-path) |
|
230 | be-path))) |
|
231 | (rocksdb-backup-engine-open opts be-path err)))) |
|
232 | ||
233 | (defun close-backup-engine-raw (be) |
|
234 | (rocksdb-backup-engine-close be)) |
|
235 | ||
236 | (defun create-new-backup-raw (be db) |
|
237 | (with-errptr (err 'rocksdb-error) |
|
238 | (rocksdb-backup-engine-create-new-backup be db err))) |
|
239 | ||
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
240 | (defun restore-from-latest-backup-raw (be db-path backup-path &optional (opt (rocksdb-restore-options-create))) |
112 | 241 | (with-errptr (err 'rocksdb-error) |
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
242 | (rocksdb-backup-engine-restore-db-from-latest-backup be db-path backup-path opt err))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
243 | |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
244 | (defun restore-from-backup-raw (be db-path backup-path backup-id &optional (opt (rocksdb-restore-options-create))) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
245 | (with-errptr (err 'rocksdb-error) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
246 | (rocksdb-backup-engine-restore-db-from-backup be db-path backup-path opt backup-id err))) |
112 | 247 | |
272
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
248 | ;;; Snapshot |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
249 | (defun create-snapshot-raw (db) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
250 | (rocksdb-create-snapshot db)) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
251 | |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
252 | (defun release-snapshot-raw (db snapshot) |
0a5e37693fdf
db protocol stuff, prep for tao, tests, with-temp-db
Richard Westhaver <ellis@rwest.io>
parents:
271
diff
changeset
|
253 | (rocksdb-release-snapshot db snapshot)) |
292
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
254 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
255 | ;;; SST |
293
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
256 | (defun create-sst-writer-raw (&optional (env-opts (rocksdb-envoptions-create)) (io-opts (rocksdb-options-create))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
257 | (rocksdb-sstfilewriter-create env-opts io-opts)) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
258 | |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
259 | (defun create-sst-writer-with-comparator-raw (comparator |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
260 | &optional |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
261 | (env-opts (rocksdb-envoptions-create)) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
262 | (io-opts (rocksdb-options-create))) |
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
263 | (rocksdb-sstfilewriter-create-with-comparator env-opts io-opts comparator)) |
292
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
264 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
265 | (defun finish-sst-writer-raw (writer) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
266 | (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
267 | (rocksdb-sstfilewriter-finish writer err))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
268 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
269 | (defun destroy-sst-writer-raw (writer) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
270 | (rocksdb-sstfilewriter-destroy writer)) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
271 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
272 | (defun open-sst-writer-raw (writer name) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
273 | (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
274 | (rocksdb-sstfilewriter-open writer name err))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
275 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
276 | ;; this function is deprecated in the Java API: |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
277 | ;; https://javadoc.io/doc/org.rocksdb/rocksdbjni/6.6.4/org/rocksdb/SstFileWriter.html |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
278 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
279 | ;; (defun sst-add-raw (writer key val) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
280 | ;; (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
281 | ;; (rocksdb-sstfilewriter-add writer key (length key) val (length val) err))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
282 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
283 | (defun sst-put-raw (writer key val) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
284 | (let ((klen (length key)) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
285 | (vlen (length val))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
286 | (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
287 | (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen)) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
288 | (v (* unsigned-char) (make-alien unsigned-char vlen))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
289 | (setfa k key) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
290 | (setfa v val) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
291 | (rocksdb-sstfilewriter-put writer k klen v vlen err))))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
292 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
293 | (defun sst-put-str-raw (writer key val) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
294 | (let ((key-octets (string-to-octets key :null-terminate nil)) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
295 | (val-octets (string-to-octets val :null-terminate nil))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
296 | (sst-put-raw writer key-octets val-octets))) |
293
e2e5c4831389
std fixes and rdb with-sst
Richard Westhaver <ellis@rwest.io>
parents:
292
diff
changeset
|
297 | |
292
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
298 | (defun sst-put-ts-raw (writer key val ts) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
299 | (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
300 | (rocksdb-sstfilewriter-put-with-ts writer key (length key) val (length val) ts (length ts) err))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
301 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
302 | (defun sst-delete-raw (writer key) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
303 | (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
304 | (rocksdb-sstfilewriter-delete writer key (length key) err))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
305 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
306 | (defun sst-delete-ts-raw (writer key ts) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
307 | (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
308 | (rocksdb-sstfilewriter-delete-with-ts writer key (length key) ts (length ts) err))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
309 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
310 | (defun sst-delete-range-raw (writer start-key end-key) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
311 | (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
312 | (rocksdb-sstfilewriter-delete-range writer start-key (length start-key) end-key (length end-key) err))) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
313 | |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
314 | (defun sst-file-size-raw (writer) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
315 | (with-errptr (err 'rocksdb-error) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
316 | (with-alien ((ret unsigned-long)) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
317 | (rocksdb-sstfilewriter-file-size writer (addr ret) err) |
00d1c8afcdbb
mostly done with std refactor, added sst-file-writer to rdb
Richard Westhaver <ellis@rwest.io>
parents:
282
diff
changeset
|
318 | ret))) |