# HG changeset patch # User Richard Westhaver # Date 1727399805 14400 # Node ID 12287fab15d07ddadaafe34eb9b0dadac26b794f # Parent 2b7d5a8d63ac67088b4c1fce142c40534043e978 rocksdb load opts and env updates diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/ffi/rocksdb/env.lisp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lisp/ffi/rocksdb/env.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -0,0 +1,18 @@ +;;; env.lisp --- RocksDB Env + +;; + +;;; Code: +(in-package :rocksdb) + +(define-alien-routine rocksdb-create-default-env (* rocksdb-env)) +(define-alien-routine rocksdb-create-mem-env (* rocksdb-env)) +(define-alien-routine rocksdb-env-set-background-threads void + (env (* rocksdb-env)) + (n int)) +(define-alien-routine rocksdb-env-get-background-threads int + (env (* rocksdb-env))) +(define-alien-routine rocksdb-env-destroy void (opt (* rocksdb-env))) +(def-with-errptr rocksdb-create-dir-if-missing void + (env (* rocksdb-env)) + (path c-string)) diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/ffi/rocksdb/macs.lisp --- a/lisp/ffi/rocksdb/macs.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/ffi/rocksdb/macs.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -18,6 +18,37 @@ (deftype rocksdb-logger-function () '(function (unsigned-byte string) (values))) +;;; Options +(defmacro with-latest-options (db-path (db-opts-var cf-names-var cf-opts-var) &body body) + ;; TODO 2024-09-26: ignore unknown? + (with-gensyms (db-opts cf-names cf-opts) + `(with-alien ((,db-opts (* rocksdb-options)) + (,cf-names (* c-string)) + (,cf-opts (* (* rocksdb-options))) + (ncols size-t) + (errptr rocksdb-errptr)) + (rocksdb-load-latest-options + ,db-path + (rocksdb-create-default-env) + t + (rocksdb-cache-create-lru 1080) + (addr ,db-opts) + (addr ncols) + (addr ,cf-names) + (addr ,cf-opts) + errptr) + (let ((,db-opts-var ,db-opts) + (,cf-names-var (coerce + (loop for i below ncols + collect (deref ,cf-names i)) + 'vector)) + (,cf-opts-var (coerce + (loop for i below ncols + collect (deref ,cf-opts i)) + 'vector))) + (unwind-protect ,@body + (rocksdb-load-latest-options-destroy ,db-opts ,cf-names ,cf-opts ncols)))))) + ;;; Merge Ops (defmacro define-full-merge-op (name &body body) `(define-alien-callable ,name (* t) diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/ffi/rocksdb/opts.lisp --- a/lisp/ffi/rocksdb/opts.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/ffi/rocksdb/opts.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -231,8 +231,12 @@ ;; TODO 2024-04-17: ;; may need to be an int -- check src (define-opt-accessor rocksdb-options use-fsync boolean) -(define-opt-accessor rocksdb-options db-log-dir c-string) -(define-opt-accessor rocksdb-options wal-dir c-string) +(define-alien-routine rocksdb-options-set-db-log-dir void + (opts (* rocksdb-options)) + (dir c-string)) +(define-alien-routine rocksdb-options-set-wal-dir void + (opts (* rocksdb-options)) + (dir c-string)) (define-opt-accessor rocksdb-options wal-ttl-seconds unsigned-long) (define-opt-accessor rocksdb-options wal-size-limit-mb unsigned-long) (define-opt-accessor rocksdb-options manifest-preallocation-size size-t) @@ -384,6 +388,18 @@ num-shard-bits memory-allocator) +#| +Load the latest rocksdb options from the specified db_path. + +On success, num_column_families will be updated with a non-zero +number indicating the number of column families. +The returned db_options, column_family_names, and column_family_options +should be released via rocksdb_load_latest_options_destroy(). + +On error, a non-null errptr that includes the error message will be +returned. db_options, column_family_names, and column_family_options +will be set to NULL. +|# (def-with-errptr rocksdb-load-latest-options void (db-path c-string) @@ -396,7 +412,7 @@ (column-family-options (* (* (* rocksdb-options))))) (define-alien-routine rocksdb-load-latest-options-destroy void - (db-options (* (* rocksdb-options))) + (db-options (* rocksdb-options)) (list-column-family-names (* c-string)) (list-column-family-options (* (* rocksdb-options))) (len size-t)) diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/ffi/rocksdb/pkg.lisp --- a/lisp/ffi/rocksdb/pkg.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/ffi/rocksdb/pkg.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -321,7 +321,14 @@ :rocksdb-put-cf-function :rocksdb-deleted-function :rocksdb-put-function - :rocksdb-open-column-families)) + :rocksdb-open-column-families + :rocksdb-create-default-env + :rocksdb-create-mem-env + :rocksdb-env-set-background-threads + :rocksdb-env-get-background-threads + :rocksdb-env-destroy + :rocksdb-create-dir-if-missing + :rocksdb-options-set-wal-dir)) (in-package :rocksdb) diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/ffi/rocksdb/rocksdb.asd --- a/lisp/ffi/rocksdb/rocksdb.asd Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/ffi/rocksdb/rocksdb.asd Thu Sep 26 21:16:45 2024 -0400 @@ -14,6 +14,7 @@ (:file "prim") (:file "types") (:file "opts") + (:file "env") (:file "sst") (:file "db") (:file "checkpoint") diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/lib/obj/pkg.lisp --- a/lisp/lib/obj/pkg.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/lib/obj/pkg.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -322,7 +322,6 @@ (:nicknames :db) (:use :cl :std :id :seq :sb-mop :sb-pcl) (:export - :load-db :get-val :set-val :dbs diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/lib/rdb/macs.lisp --- a/lisp/lib/rdb/macs.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/lib/rdb/macs.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -14,9 +14,7 @@ (lambda (c) (handle-errptr ,e ,errtyp ,params)))) (progn ,@body)) - (handle-errptr ,e ,errtyp ,params) - ))) - + (handle-errptr ,e ,errtyp ,params)))) ;;; opts (defmacro rdb-opt-setter (key) @@ -88,11 +86,11 @@ (rocksdb-backup-engine-close ,be-var)))) ;;; top-level +;; TODO 2024-09-26: (defmacro do-db ((db opts) accessors &body body) "Database Iteration construct. OPTS are used to provide top-level options dynamically bound to DB. ACCESSORS is a list of database - accessors which are available to call in BODY." - ) + accessors which are available to call in BODY.") ;;; temp-db (defvar *temp-db-path-generator* @@ -139,3 +137,19 @@ ,@(when file `((open-sst ,sst ,file))) ,@body ,@(when destroy `((destroy-sst ,sst))))) + +;;; opts +(defmacro with-latest-opts ((db-var db-path) &body body) + `(rocksdb::with-latest-options ,(string db-path) (db-opts cf-names cf-opts) + (let ((opts (make-rdb-opts))) + (setf (rdb-opts-sap opts) db-opts) + (let ((cfs (coerce + (loop for name across cf-names + for opt across cf-opts + collect + (let ((cf-opts (make-rdb-opts))) + (setf (rdb-opts-sap cf-opts) opt) + (make-rdb-cf name :opts cf-opts))) + 'vector))) + (let ((,db-var (make-rdb ,db-path opts cfs))) + ,@body))))) diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/lib/rdb/obj.lisp --- a/lisp/lib/rdb/obj.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/lib/rdb/obj.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -159,7 +159,7 @@ tslen))) ;;; column family -(defstruct (rdb-cf (:constructor make-rdb-cf (name &key key-type val-type sap))) +(defstruct (rdb-cf (:constructor make-rdb-cf (name &key opts key-type val-type sap))) "RDB Column Family structure. Contains a name, key-type, val-type, and a system-area-pointer to the underlying rocksdb_cf_t handle. @@ -167,6 +167,7 @@ 'octet-vector. This is needed to distinguish the value 'octet-vector being supplied by the user from the default value." (name "" :type string) + (opts (default-rdb-opts) :type rdb-opts) (key-type nil :type (or list symbol)) (val-type nil :type (or list symbol)) (sap nil :type (or null alien))) @@ -380,12 +381,12 @@ (defmethod create-cf ((db rdb) (cf rdb-cf)) (create-cf-raw (rdb-db db) (rdb-cf-name cf) (rdb-opts-sap (rdb-opts db)))) -(defmethod open-cf ((db rdb) (cf rdb-cf) &optional error) +(defmethod open-cf ((db rdb) (cf rdb-cf) &optional (error t)) (unless (null (rdb-cf-sap cf)) (if error (rdb-error "column family is already open - close before re-opening.") cf) - (setf (rdb-cf-sap cf) (open-cf-raw (rdb-db db) (default-rocksdb-options) (rdb-cf-name cf))))) + (setf (rdb-cf-sap cf) (open-cf-raw (rdb-db db) (rdb-cf-opts cf) (rdb-cf-name cf))))) (defmethod open-cf ((db rdb) (cf string) &optional (error t)) (if-let ((cf (find-cf cf db))) diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/lib/rdb/pkg.lisp --- a/lisp/lib/rdb/pkg.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/lib/rdb/pkg.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -120,7 +120,9 @@ :rdb-cf-val-type :close-cf :open-cf - :close-cfs)) + :close-cfs + :rdb-cf-opts + :with-latest-opts)) (in-package :rdb) (rocksdb:load-rocksdb nil) diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/lib/rdb/proto.lisp --- a/lisp/lib/rdb/proto.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/lib/rdb/proto.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -36,7 +36,7 @@ (:documentation "Implicitly pull foreign values from the sap associated with SELF.")) (defgeneric push-cf (cf self) (:documentation "Push a column-family to a sap.")) -(defgeneric open-cf (self cf &optional error) +(defgeneric open-cf (self cf &optional opts) (:documentation "Open column-family CF in SELF. When ERROR is non-nil signal an error if the column-family is already open.")) (defgeneric open-cfs (self) diff -r 2b7d5a8d63ac -r 12287fab15d0 lisp/lib/rt/tracing.lisp --- a/lisp/lib/rt/tracing.lisp Wed Sep 25 21:39:39 2024 -0400 +++ b/lisp/lib/rt/tracing.lisp Thu Sep 26 21:16:45 2024 -0400 @@ -264,7 +264,7 @@ (symbol-function 'sb-debug::trace-end-breakpoint-fun) *original-trace-end-breakpoint-fun*) (sb-ext:lock-package (find-package 'sb-debug))) -(defun start-tracing (&rest specs) +(defun start-tracing (specs) (install-tracing-overrides) `(progn (trace :encapsulate t :methods t ,@specs))) @@ -317,7 +317,7 @@ `(unwind-protect (progn (reset-tracing) - (start-tracing ,@specs) + (start-tracing ',specs) (progn ,@body)) (stop-tracing)))