changelog shortlog graph tags branches changeset files revisions annotate raw help

Mercurial > core / lisp/lib/rdb/raw.lisp

changeset 624: 97dd03beda03
parent: e2e5c4831389
child: 5f88b237ce29
author: Richard Westhaver <ellis@rwest.io>
date: Sun, 25 Aug 2024 20:28:57 -0400
permissions: -rw-r--r--
description: rocksdb updates in coordination with infra/scripts/org-graph-db-init.lisp 
     1 ;;; rdb/raw.lisp --- Raw wrappers for ROCKSDB alien interface
     2 
     3 ;;; Code:
     4 (in-package :rdb)
     5 
     6 ;;; Options
     7 (defun make-rocksdb-options (&optional init-fn)
     8   "Make and return RDB-OPTS. INIT-FN is an optional argument which must be a
     9 lambda which takes a single parameter (the RDB-OPTS sap). It is used
    10 to initialize the instance with custom configuration."
    11   (let ((opts (rocksdb-options-create)))
    12     (when init-fn (funcall init-fn opts))
    13     opts))
    14 
    15 (defun default-rocksdb-options ()
    16   (make-rocksdb-options
    17    (lambda (o) (rocksdb-options-set-create-if-missing o t))))
    18 
    19 (defun get-stats-raw (opt htype)
    20   (with-alien ((hist (* rocksdb-statistics-histogram-data) (rocksdb-statistics-histogram-data-create)))
    21     (rocksdb-options-statistics-get-histogram-data opt htype hist)
    22     (deref hist)))
    23 
    24 ;;; DB
    25 (defun open-db-raw (db-path &optional (opts (default-rocksdb-options)))
    26   (with-errptr (err 'open-db-error (list :db db-path))
    27     (let* ((db-path (if (pathnamep db-path)
    28                         (namestring db-path)
    29                         db-path)))
    30       (rocksdb-open opts db-path err))))
    31 
    32 (defun close-db-raw (db)
    33   (rocksdb-close db))
    34 
    35 (defun destroy-db-raw (path &optional (opt (rocksdb-options-create)))
    36   (with-errptr (err 'destroy-db-error (list :db path))
    37     (rocksdb-destroy-db opt (namestring (uiop:ensure-directory-pathname path)) err)
    38     (rocksdb-options-destroy opt)))
    39 
    40 (defun get-property-raw (db str)
    41   (rocksdb-property-value db (make-alien-string str)))
    42 
    43 (defun get-metadata-raw (db &optional cf)
    44   (if cf
    45       (rocksdb-get-column-family-metadata-cf db cf)
    46       (rocksdb-get-column-family-metadata db)))
    47 
    48 (defun flush-db-raw (db &optional (opts (rocksdb-flushoptions-create)))
    49   (with-errptr (err 'flush-db-error (list :db db))
    50     (rocksdb-flush db opts err)))
    51 
    52 (defun repair-db-raw (name &optional (opts (rocksdb-options-create)))
    53   (with-errptr (err 'repair-db-error (list :name name))
    54     (rocksdb-repair-db opts name err)))
    55 
    56 (defun ingest-db-raw (db files &optional (opts (rocksdb-ingestexternalfileoptions-create)))
    57   (let ((flen (length files)))
    58     (with-errptr (err 'ingest-db-error)
    59       (with-alien ((flist (* c-string) (make-alien c-string flen)))
    60         (loop for f in files
    61               for i from 0 to flen
    62               do (setf (deref flist i) (make-alien-string f :null-terminate nil)))
    63         (rocksdb-ingest-external-file db flist flen opts err)))))
    64 
    65 (defun ingest-db-cf-raw (db cf files &optional (opts (rocksdb-ingestexternalfileoptions-create)))
    66   (let ((flen (length files)))
    67     (with-errptr (err 'ingest-db-error)
    68       (with-alien ((flist (* c-string) (make-alien c-string flen)))
    69         (loop for f in files
    70               for i from 0 to flen
    71               do (setf (deref flist i) (make-alien-string f :null-terminate nil)))
    72         (rocksdb-ingest-external-file-cf db cf flist flen opts err)))))
    73   
    74 ;;; KVs
    75 (defun put-kv-raw (db key val &optional (opts (rocksdb-writeoptions-create)))
    76   (let ((klen (length key))
    77 	(vlen (length val)))
    78     (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen))
    79 		 (v (* unsigned-char) (make-alien unsigned-char vlen)))
    80       (setfa k key)
    81       (setfa v val)
    82       (with-errptr (err 'put-kv-error (list :db db :kv (cons key val)))
    83         (rocksdb-put db
    84 		     opts
    85 		     k
    86 		     klen
    87 		     v
    88 		     vlen
    89 		     err)))))
    90 
    91 (defun put-kv-str-raw (db key val &optional (opts (rocksdb-writeoptions-create)))
    92   (let ((key-octets (string-to-octets key :null-terminate nil))
    93         (val-octets (string-to-octets val :null-terminate nil)))
    94     (put-kv-raw db key-octets val-octets opts)))
    95 
    96 (defun get-kv-raw (db key &optional (opt (rocksdb-readoptions-create)))
    97   (let ((klen (length key)))
    98     (with-errptr (err 'get-kv-error (list :db db :key key))
    99       (with-alien ((vlen size-t)
   100 		   (k (* unsigned-char) (make-alien unsigned-char klen)))
   101         (setfa k key)
   102         (let* ((val (rocksdb-get db
   103 			         opt
   104 			         k klen
   105                                  (addr vlen)
   106 			         err)))
   107 	  ;; helps if we know the vlen beforehand, would need a custom
   108 	  ;; C-side function probably.
   109           (let ((v (make-array vlen :element-type 'octet)))
   110             (clone-octets-from-alien val v vlen)
   111             (coerce v 'octet-vector)))))))
   112 
   113 (defun get-kv-str-raw (db key &optional (opt (rocksdb-readoptions-create)))
   114   (let ((k (string-to-octets key)))
   115     (let ((v (get-kv-raw db k opt)))
   116       (when v (octets-to-string v)))))
   117 
   118 ;;; Column Family
   119 (defun open-cf-raw (db name &optional (opt (rocksdb-options-create)))
   120   (with-errptr (err 'rocksdb-cf-error (list :db db :cf name))
   121     (rocksdb-open-column-families opt name 1 nil nil nil err)))
   122 
   123 (defun create-cf-raw (db name &optional (opt (rocksdb-options-create)))
   124   (with-errptr (err 'rocksdb-cf-error (list :db db :cf name)) 
   125     (rocksdb-create-column-family db opt name err)))
   126 
   127 (defun destroy-cf-raw (cf)
   128   (rocksdb-column-family-handle-destroy cf))
   129 
   130 (defun get-cf-raw (db cf key &optional (opt (rocksdb-readoptions-create)))
   131   (let ((klen (length key)))
   132     (with-errptr (err 'get-kv-error (list :db db :key key))
   133       (with-alien ((vlen (* size-t) (make-alien size-t 0))
   134 		   (k (* unsigned-char) (make-alien unsigned-char klen)))
   135         (setfa k key)
   136         (let* ((val (rocksdb-get-cf db
   137 			            opt
   138                                     cf
   139 			            k klen
   140                                     vlen
   141 			            err)))
   142 	  ;; helps if we know the vlen beforehand, would need a custom
   143 	  ;; C-side function probably.
   144 	  (let ((v (make-array (deref vlen) :element-type 'unsigned-byte)))
   145             (clone-octets-from-alien val v (deref vlen))
   146 	    v))))))
   147 
   148 (defun get-cf-str-raw (db cf key &optional (opt (rocksdb-readoptions-create)))
   149   (let ((k (string-to-octets key :null-terminate nil)))
   150     (let ((v (get-cf-raw db cf k opt)))
   151       (when v (concatenate 'string (map 'vector #'code-char v))))))
   152 
   153 (defun put-cf-raw (db cf key val &optional (opts (rocksdb-writeoptions-create)))
   154   (let ((klen (length key))
   155         (vlen (length val)))
   156     (with-errptr (err 'put-kv-error (list :db db :kv (cons key val)))
   157       (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen))
   158                    (v (* unsigned-char) (make-alien unsigned-char vlen)))
   159         (setfa k key)
   160         (setfa v val)
   161         (rocksdb-put-cf db
   162                         opts
   163                         cf
   164                         k klen
   165                         v vlen
   166                         err)))))
   167 
   168 (defun put-cf-str-raw (db cf key val &optional (opt (rocksdb-writeoptions-create)))
   169   (let ((key-octets (string-to-octets key :null-terminate nil))
   170         (val-octets (string-to-octets val :null-terminate nil)))
   171     (put-cf-raw db cf key-octets val-octets opt)))
   172 
   173 ;;; Iterators
   174 (defun create-iter-raw (db &optional (opt (rocksdb-readoptions-create)))
   175   (rocksdb-create-iterator db opt))
   176 
   177 (defun create-cf-iter-raw (db cf &optional (opt (rocksdb-readoptions-create)))
   178   (rocksdb-create-iterator-cf db opt cf))
   179 
   180 (defun destroy-iter-raw (iter)
   181   (rocksdb-iter-destroy iter))
   182 
   183 (defun iter-key-raw (iter)
   184   (with-alien ((klen-ptr (* size-t) (make-alien size-t 0)))
   185     (let* ((key-ptr (rocksdb-iter-key iter klen-ptr))
   186            (klen (deref klen-ptr))
   187            (k (make-array klen :element-type '(unsigned-byte 8))))
   188       (clone-octets-from-alien key-ptr k klen)
   189       k)))
   190 
   191 (defun iter-key-str-raw (iter)
   192   (when-let ((k (iter-key-raw iter)))
   193     (octets-to-string k)))
   194 
   195 (defun iter-val-raw (iter)
   196   (with-alien ((vlen-ptr (* size-t) (make-alien size-t 0)))
   197     (let* ((val-ptr (rocksdb-iter-value iter vlen-ptr))
   198            (vlen (deref vlen-ptr))
   199            (v (make-array vlen :element-type '(unsigned-byte 8))))
   200       (clone-octets-from-alien val-ptr v vlen)
   201       v)))
   202 
   203 (defun iter-val-str-raw (iter)
   204   (when-let ((v (iter-val-raw iter)))
   205     (octets-to-string v)))
   206 
   207 ;;; Backup Engine
   208 (defun open-backup-engine-raw (be-path &optional (opts (rocksdb-options-create)))
   209   (with-errptr (err 'open-backup-engine-error (list :db be-path))
   210     (let ((be-path (if (pathnamep be-path)
   211                        (namestring be-path)
   212                        be-path)))
   213       (rocksdb-backup-engine-open opts be-path err))))
   214 
   215 (defun close-backup-engine-raw (be)
   216   (rocksdb-backup-engine-close be))
   217 
   218 (defun create-new-backup-raw (be db)
   219   (with-errptr (err 'rocksdb-error)
   220     (rocksdb-backup-engine-create-new-backup be db err)))
   221 
   222 (defun restore-from-latest-backup-raw (be db-path backup-path &optional (opt (rocksdb-restore-options-create)))
   223   (with-errptr (err 'rocksdb-error)
   224     (rocksdb-backup-engine-restore-db-from-latest-backup be db-path backup-path opt err)))
   225 
   226 (defun restore-from-backup-raw (be db-path backup-path backup-id &optional (opt (rocksdb-restore-options-create)))
   227   (with-errptr (err 'rocksdb-error)
   228     (rocksdb-backup-engine-restore-db-from-backup be db-path backup-path opt backup-id err)))
   229 
   230 ;;; Snapshot
   231 (defun create-snapshot-raw (db)
   232   (rocksdb-create-snapshot db))
   233 
   234 (defun release-snapshot-raw (db snapshot)
   235   (rocksdb-release-snapshot db snapshot))
   236 
   237 ;;; SST
   238 (defun create-sst-writer-raw (&optional (env-opts (rocksdb-envoptions-create)) (io-opts (rocksdb-options-create)))
   239   (rocksdb-sstfilewriter-create env-opts io-opts))
   240 
   241 (defun create-sst-writer-with-comparator-raw (comparator
   242                                               &optional
   243                                                 (env-opts (rocksdb-envoptions-create))
   244                                                 (io-opts (rocksdb-options-create)))
   245   (rocksdb-sstfilewriter-create-with-comparator env-opts io-opts comparator))
   246 
   247 (defun finish-sst-writer-raw (writer)
   248   (with-errptr (err 'rocksdb-error)
   249     (rocksdb-sstfilewriter-finish writer err)))
   250 
   251 (defun destroy-sst-writer-raw (writer)
   252   (rocksdb-sstfilewriter-destroy writer))
   253 
   254 (defun open-sst-writer-raw (writer name)
   255   (with-errptr (err 'rocksdb-error)
   256     (rocksdb-sstfilewriter-open writer name err)))
   257 
   258 ;; this function is deprecated in the Java API:
   259 ;; https://javadoc.io/doc/org.rocksdb/rocksdbjni/6.6.4/org/rocksdb/SstFileWriter.html
   260 
   261 ;; (defun sst-add-raw (writer key val)
   262 ;;   (with-errptr (err 'rocksdb-error)
   263 ;;     (rocksdb-sstfilewriter-add writer key (length key) val (length val) err)))
   264 
   265 (defun sst-put-raw (writer key val)
   266   (let ((klen (length key))
   267         (vlen (length val)))
   268     (with-errptr (err 'rocksdb-error)
   269       (with-alien ((k (* unsigned-char) (make-alien unsigned-char klen))
   270                    (v (* unsigned-char) (make-alien unsigned-char vlen)))
   271         (setfa k key)
   272         (setfa v val)
   273         (rocksdb-sstfilewriter-put writer k klen v vlen err)))))
   274 
   275 (defun sst-put-str-raw (writer key val)
   276   (let ((key-octets (string-to-octets key :null-terminate nil))
   277         (val-octets (string-to-octets val :null-terminate nil)))
   278     (sst-put-raw writer key-octets val-octets)))
   279 
   280 (defun sst-put-ts-raw (writer key val ts)
   281   (with-errptr (err 'rocksdb-error)
   282     (rocksdb-sstfilewriter-put-with-ts writer key (length key) val (length val) ts (length ts) err)))
   283 
   284 (defun sst-delete-raw (writer key)
   285   (with-errptr (err 'rocksdb-error)
   286     (rocksdb-sstfilewriter-delete writer key (length key) err)))
   287 
   288 (defun sst-delete-ts-raw (writer key ts)
   289   (with-errptr (err 'rocksdb-error)
   290     (rocksdb-sstfilewriter-delete-with-ts writer key (length key) ts (length ts) err)))
   291 
   292 (defun sst-delete-range-raw (writer start-key end-key)
   293   (with-errptr (err 'rocksdb-error)
   294     (rocksdb-sstfilewriter-delete-range writer start-key (length start-key) end-key (length end-key) err)))
   295 
   296 (defun sst-file-size-raw (writer)
   297   (with-errptr (err 'rocksdb-error)
   298     (with-alien ((ret unsigned-long))
   299       (rocksdb-sstfilewriter-file-size writer (addr ret) err)
   300       ret)))