1.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2+++ b/lisp/ffi/rocksdb/env.lisp Thu Sep 26 21:16:45 2024 -0400
1.3@@ -0,0 +1,18 @@
1.4+;;; env.lisp --- RocksDB Env
1.5+
1.6+;;
1.7+
1.8+;;; Code:
1.9+(in-package :rocksdb)
1.10+
1.11+(define-alien-routine rocksdb-create-default-env (* rocksdb-env))
1.12+(define-alien-routine rocksdb-create-mem-env (* rocksdb-env))
1.13+(define-alien-routine rocksdb-env-set-background-threads void
1.14+ (env (* rocksdb-env))
1.15+ (n int))
1.16+(define-alien-routine rocksdb-env-get-background-threads int
1.17+ (env (* rocksdb-env)))
1.18+(define-alien-routine rocksdb-env-destroy void (opt (* rocksdb-env)))
1.19+(def-with-errptr rocksdb-create-dir-if-missing void
1.20+ (env (* rocksdb-env))
1.21+ (path c-string))
2.1--- a/lisp/ffi/rocksdb/macs.lisp Wed Sep 25 21:39:39 2024 -0400
2.2+++ b/lisp/ffi/rocksdb/macs.lisp Thu Sep 26 21:16:45 2024 -0400
2.3@@ -18,6 +18,37 @@
2.4 (deftype rocksdb-logger-function ()
2.5 '(function (unsigned-byte string) (values)))
2.6
2.7+;;; Options
2.8+(defmacro with-latest-options (db-path (db-opts-var cf-names-var cf-opts-var) &body body)
2.9+ ;; TODO 2024-09-26: ignore unknown?
2.10+ (with-gensyms (db-opts cf-names cf-opts)
2.11+ `(with-alien ((,db-opts (* rocksdb-options))
2.12+ (,cf-names (* c-string))
2.13+ (,cf-opts (* (* rocksdb-options)))
2.14+ (ncols size-t)
2.15+ (errptr rocksdb-errptr))
2.16+ (rocksdb-load-latest-options
2.17+ ,db-path
2.18+ (rocksdb-create-default-env)
2.19+ t
2.20+ (rocksdb-cache-create-lru 1080)
2.21+ (addr ,db-opts)
2.22+ (addr ncols)
2.23+ (addr ,cf-names)
2.24+ (addr ,cf-opts)
2.25+ errptr)
2.26+ (let ((,db-opts-var ,db-opts)
2.27+ (,cf-names-var (coerce
2.28+ (loop for i below ncols
2.29+ collect (deref ,cf-names i))
2.30+ 'vector))
2.31+ (,cf-opts-var (coerce
2.32+ (loop for i below ncols
2.33+ collect (deref ,cf-opts i))
2.34+ 'vector)))
2.35+ (unwind-protect ,@body
2.36+ (rocksdb-load-latest-options-destroy ,db-opts ,cf-names ,cf-opts ncols))))))
2.37+
2.38 ;;; Merge Ops
2.39 (defmacro define-full-merge-op (name &body body)
2.40 `(define-alien-callable ,name (* t)
3.1--- a/lisp/ffi/rocksdb/opts.lisp Wed Sep 25 21:39:39 2024 -0400
3.2+++ b/lisp/ffi/rocksdb/opts.lisp Thu Sep 26 21:16:45 2024 -0400
3.3@@ -231,8 +231,12 @@
3.4 ;; TODO 2024-04-17:
3.5 ;; may need to be an int -- check src
3.6 (define-opt-accessor rocksdb-options use-fsync boolean)
3.7-(define-opt-accessor rocksdb-options db-log-dir c-string)
3.8-(define-opt-accessor rocksdb-options wal-dir c-string)
3.9+(define-alien-routine rocksdb-options-set-db-log-dir void
3.10+ (opts (* rocksdb-options))
3.11+ (dir c-string))
3.12+(define-alien-routine rocksdb-options-set-wal-dir void
3.13+ (opts (* rocksdb-options))
3.14+ (dir c-string))
3.15 (define-opt-accessor rocksdb-options wal-ttl-seconds unsigned-long)
3.16 (define-opt-accessor rocksdb-options wal-size-limit-mb unsigned-long)
3.17 (define-opt-accessor rocksdb-options manifest-preallocation-size size-t)
3.18@@ -384,6 +388,18 @@
3.19 num-shard-bits
3.20 memory-allocator)
3.21
3.22+#|
3.23+Load the latest rocksdb options from the specified db_path.
3.24+
3.25+On success, num_column_families will be updated with a non-zero
3.26+number indicating the number of column families.
3.27+The returned db_options, column_family_names, and column_family_options
3.28+should be released via rocksdb_load_latest_options_destroy().
3.29+
3.30+On error, a non-null errptr that includes the error message will be
3.31+returned. db_options, column_family_names, and column_family_options
3.32+will be set to NULL.
3.33+|#
3.34 (def-with-errptr rocksdb-load-latest-options
3.35 void
3.36 (db-path c-string)
3.37@@ -396,7 +412,7 @@
3.38 (column-family-options (* (* (* rocksdb-options)))))
3.39
3.40 (define-alien-routine rocksdb-load-latest-options-destroy void
3.41- (db-options (* (* rocksdb-options)))
3.42+ (db-options (* rocksdb-options))
3.43 (list-column-family-names (* c-string))
3.44 (list-column-family-options (* (* rocksdb-options)))
3.45 (len size-t))
4.1--- a/lisp/ffi/rocksdb/pkg.lisp Wed Sep 25 21:39:39 2024 -0400
4.2+++ b/lisp/ffi/rocksdb/pkg.lisp Thu Sep 26 21:16:45 2024 -0400
4.3@@ -321,7 +321,14 @@
4.4 :rocksdb-put-cf-function
4.5 :rocksdb-deleted-function
4.6 :rocksdb-put-function
4.7- :rocksdb-open-column-families))
4.8+ :rocksdb-open-column-families
4.9+ :rocksdb-create-default-env
4.10+ :rocksdb-create-mem-env
4.11+ :rocksdb-env-set-background-threads
4.12+ :rocksdb-env-get-background-threads
4.13+ :rocksdb-env-destroy
4.14+ :rocksdb-create-dir-if-missing
4.15+ :rocksdb-options-set-wal-dir))
4.16
4.17 (in-package :rocksdb)
4.18
5.1--- a/lisp/ffi/rocksdb/rocksdb.asd Wed Sep 25 21:39:39 2024 -0400
5.2+++ b/lisp/ffi/rocksdb/rocksdb.asd Thu Sep 26 21:16:45 2024 -0400
5.3@@ -14,6 +14,7 @@
5.4 (:file "prim")
5.5 (:file "types")
5.6 (:file "opts")
5.7+ (:file "env")
5.8 (:file "sst")
5.9 (:file "db")
5.10 (:file "checkpoint")
6.1--- a/lisp/lib/obj/pkg.lisp Wed Sep 25 21:39:39 2024 -0400
6.2+++ b/lisp/lib/obj/pkg.lisp Thu Sep 26 21:16:45 2024 -0400
6.3@@ -322,7 +322,6 @@
6.4 (:nicknames :db)
6.5 (:use :cl :std :id :seq :sb-mop :sb-pcl)
6.6 (:export
6.7- :load-db
6.8 :get-val
6.9 :set-val
6.10 :dbs
7.1--- a/lisp/lib/rdb/macs.lisp Wed Sep 25 21:39:39 2024 -0400
7.2+++ b/lisp/lib/rdb/macs.lisp Thu Sep 26 21:16:45 2024 -0400
7.3@@ -14,9 +14,7 @@
7.4 (lambda (c)
7.5 (handle-errptr ,e ,errtyp ,params))))
7.6 (progn ,@body))
7.7- (handle-errptr ,e ,errtyp ,params)
7.8- )))
7.9-
7.10+ (handle-errptr ,e ,errtyp ,params))))
7.11
7.12 ;;; opts
7.13 (defmacro rdb-opt-setter (key)
7.14@@ -88,11 +86,11 @@
7.15 (rocksdb-backup-engine-close ,be-var))))
7.16
7.17 ;;; top-level
7.18+;; TODO 2024-09-26:
7.19 (defmacro do-db ((db opts) accessors &body body)
7.20 "Database Iteration construct. OPTS are used to provide top-level
7.21 options dynamically bound to DB. ACCESSORS is a list of database
7.22- accessors which are available to call in BODY."
7.23- )
7.24+ accessors which are available to call in BODY.")
7.25
7.26 ;;; temp-db
7.27 (defvar *temp-db-path-generator*
7.28@@ -139,3 +137,19 @@
7.29 ,@(when file `((open-sst ,sst ,file)))
7.30 ,@body
7.31 ,@(when destroy `((destroy-sst ,sst)))))
7.32+
7.33+;;; opts
7.34+(defmacro with-latest-opts ((db-var db-path) &body body)
7.35+ `(rocksdb::with-latest-options ,(string db-path) (db-opts cf-names cf-opts)
7.36+ (let ((opts (make-rdb-opts)))
7.37+ (setf (rdb-opts-sap opts) db-opts)
7.38+ (let ((cfs (coerce
7.39+ (loop for name across cf-names
7.40+ for opt across cf-opts
7.41+ collect
7.42+ (let ((cf-opts (make-rdb-opts)))
7.43+ (setf (rdb-opts-sap cf-opts) opt)
7.44+ (make-rdb-cf name :opts cf-opts)))
7.45+ 'vector)))
7.46+ (let ((,db-var (make-rdb ,db-path opts cfs)))
7.47+ ,@body)))))
8.1--- a/lisp/lib/rdb/obj.lisp Wed Sep 25 21:39:39 2024 -0400
8.2+++ b/lisp/lib/rdb/obj.lisp Thu Sep 26 21:16:45 2024 -0400
8.3@@ -159,7 +159,7 @@
8.4 tslen)))
8.5
8.6 ;;; column family
8.7-(defstruct (rdb-cf (:constructor make-rdb-cf (name &key key-type val-type sap)))
8.8+(defstruct (rdb-cf (:constructor make-rdb-cf (name &key opts key-type val-type sap)))
8.9 "RDB Column Family structure. Contains a name, key-type, val-type,
8.10 and a system-area-pointer to the underlying rocksdb_cf_t handle.
8.11
8.12@@ -167,6 +167,7 @@
8.13 'octet-vector. This is needed to distinguish the value 'octet-vector being
8.14 supplied by the user from the default value."
8.15 (name "" :type string)
8.16+ (opts (default-rdb-opts) :type rdb-opts)
8.17 (key-type nil :type (or list symbol))
8.18 (val-type nil :type (or list symbol))
8.19 (sap nil :type (or null alien)))
8.20@@ -380,12 +381,12 @@
8.21 (defmethod create-cf ((db rdb) (cf rdb-cf))
8.22 (create-cf-raw (rdb-db db) (rdb-cf-name cf) (rdb-opts-sap (rdb-opts db))))
8.23
8.24-(defmethod open-cf ((db rdb) (cf rdb-cf) &optional error)
8.25+(defmethod open-cf ((db rdb) (cf rdb-cf) &optional (error t))
8.26 (unless (null (rdb-cf-sap cf))
8.27 (if error
8.28 (rdb-error "column family is already open - close before re-opening.")
8.29 cf)
8.30- (setf (rdb-cf-sap cf) (open-cf-raw (rdb-db db) (default-rocksdb-options) (rdb-cf-name cf)))))
8.31+ (setf (rdb-cf-sap cf) (open-cf-raw (rdb-db db) (rdb-cf-opts cf) (rdb-cf-name cf)))))
8.32
8.33 (defmethod open-cf ((db rdb) (cf string) &optional (error t))
8.34 (if-let ((cf (find-cf cf db)))
9.1--- a/lisp/lib/rdb/pkg.lisp Wed Sep 25 21:39:39 2024 -0400
9.2+++ b/lisp/lib/rdb/pkg.lisp Thu Sep 26 21:16:45 2024 -0400
9.3@@ -120,7 +120,9 @@
9.4 :rdb-cf-val-type
9.5 :close-cf
9.6 :open-cf
9.7- :close-cfs))
9.8+ :close-cfs
9.9+ :rdb-cf-opts
9.10+ :with-latest-opts))
9.11
9.12 (in-package :rdb)
9.13 (rocksdb:load-rocksdb nil)
10.1--- a/lisp/lib/rdb/proto.lisp Wed Sep 25 21:39:39 2024 -0400
10.2+++ b/lisp/lib/rdb/proto.lisp Thu Sep 26 21:16:45 2024 -0400
10.3@@ -36,7 +36,7 @@
10.4 (:documentation "Implicitly pull foreign values from the sap associated with SELF."))
10.5 (defgeneric push-cf (cf self)
10.6 (:documentation "Push a column-family to a sap."))
10.7-(defgeneric open-cf (self cf &optional error)
10.8+(defgeneric open-cf (self cf &optional opts)
10.9 (:documentation "Open column-family CF in SELF. When ERROR is non-nil signal an error if the
10.10 column-family is already open."))
10.11 (defgeneric open-cfs (self)
11.1--- a/lisp/lib/rt/tracing.lisp Wed Sep 25 21:39:39 2024 -0400
11.2+++ b/lisp/lib/rt/tracing.lisp Thu Sep 26 21:16:45 2024 -0400
11.3@@ -264,7 +264,7 @@
11.4 (symbol-function 'sb-debug::trace-end-breakpoint-fun) *original-trace-end-breakpoint-fun*)
11.5 (sb-ext:lock-package (find-package 'sb-debug)))
11.6
11.7-(defun start-tracing (&rest specs)
11.8+(defun start-tracing (specs)
11.9 (install-tracing-overrides)
11.10 `(progn
11.11 (trace :encapsulate t :methods t ,@specs)))
11.12@@ -317,7 +317,7 @@
11.13 `(unwind-protect
11.14 (progn
11.15 (reset-tracing)
11.16- (start-tracing ,@specs)
11.17+ (start-tracing ',specs)
11.18 (progn
11.19 ,@body))
11.20 (stop-tracing)))
