1.1--- a/lisp/ffi/rocksdb/db.lisp Sun Dec 10 23:02:43 2023 -0500
1.2+++ b/lisp/ffi/rocksdb/db.lisp Mon Dec 11 22:05:25 2023 -0500
1.3@@ -246,4 +246,88 @@
1.4 (define-alien-routine rocksdb-transactiondb-close void
1.5 (tdb (* rocksdb-transactiondb)))
1.6
1.7+;;; Perfcontext
1.8+(define-alien-routine rocksdb-set-perf-level void (val int))
1.9+
1.10+(define-alien-routine rocksdb-perfcontext-create (* rocksdb-perfcontext))
1.11+
1.12+(define-alien-routine rocksdb-perfcontext-reset void (* rocksdb-perfcontext))
1.13+
1.14+(define-alien-routine rocksdb-perfcontext-report (* char)
1.15+ (context (* rocksdb-perfcontext))
1.16+ (exclude-zero-counters unsigned-char))
1.17+
1.18+(define-alien-routine rocksdb-perfcontext-metric unsigned-long
1.19+ (context (* rocksdb-perfcontext)) (metric int))
1.20+
1.21+(define-alien-routine rocksdb-perfcontext-destroy void (* rocksdb-perfcontext))
1.22+
1.23+;;; Compaction Filter
1.24+;; (define-alien-routine rocksdb-compactionfilter-create (* rocksdb-compactionfilter)
1.25+;; (state (* void))
1.26+;; (destructor (* void))
1.27+;; (filter (* unsigned-char))
1.28+;; (name (* char)))
1.29+
1.30+(define-alien-routine rocksdb-compactionfilter-set-ignore-snapshots void
1.31+ (self (* rocksdb-compactionfilter)) (val unsigned-char))
1.32+
1.33+(define-alien-routine rocksdb-compactionfilter-destroy void
1.34+ (self (* rocksdb-compactionfilter)))
1.35+
1.36+;;; Compaction Filter Context
1.37+(define-alien-routine rocksdb-compactionfiltercontext-is-full-compaction unsigned-char
1.38+ (context (* rocksdb-compactionfiltercontext)))
1.39+
1.40+(define-alien-routine rocksdb-compactionfiltercontext-is-manual-compaction unsigned-char
1.41+ (context (* rocksdb-compactionfiltercontext)))
1.42+
1.43+;;; Compaction Filter Factory
1.44+
1.45+;;; Comparator
1.46+;; TODO 2023-12-11:
1.47+;; (define-alien-routine rocksdb-comparator-create (* rocksdb-comparator)
1.48+;; (state (* void))
1.49+;; (destructor (* void))
1.50+;; (compare (* int))
1.51+;; (name (* char)))
1.52+
1.53+(define-alien-routine rocksdb-comparator-destroy void (self (* rocksdb-comparator)))
1.54+
1.55+;; (define-alien-routine rocksdb-comparator-with-ts-create (* rocksdb-comparator)
1.56+;; (state (* void))
1.57+;; (destructor (* void))
1.58+;; (compare (* int))
1.59+;; (compare-ts (* int))
1.60+;; (compare-without-ts (* int))
1.61+;; (name (* char)))
1.62+
1.63+;;; Filter Policy
1.64+(define-alien-routine rocksdb-filterpolicy-destroy void (self (* rocksdb-filterpolicy)))
1.65+
1.66+(define-alien-routine rocksdb-filterpolicy-create-bloom (* rocksdb-filterpolicy)
1.67+ (bits-per-key double))
1.68+
1.69+(define-alien-routine rocksdb-filterpolicy-create-bloom-full (* rocksdb-filterpolicy)
1.70+ (bits-per-key double))
1.71+
1.72+(define-alien-routine rocksdb-filterpolicy-create-ribbon (* rocksdb-filterpolicy)
1.73+ (bloom-equivalent-bits-per-key double))
1.74+
1.75+(define-alien-routine rocksdb-filterpolicy-create-ribbon-hybrid (* rocksdb-filterpolicy)
1.76+ (bloom-equivalent-bits-per-key double)
1.77+ (bloom-before-level int))
1.78+
1.79+;;; Merge Operator
1.80+;; TODO 2023-12-11:
1.81+;; (define-alien-routine rocksdb-mergeoperator-create (* rocksdb-mergeoperator)
1.82+;; (state (* void))
1.83+;; (destructor (* void))
1.84+;; (full-merge (* char))
1.85+;; (partial-merge (* char))
1.86+;; (delete-value (* void))
1.87+;; (name (* char)))
1.88+
1.89+(define-alien-routine rocksdb-mergeoperator-destroy void (self (* rocksdb-mergeoperator)))
1.90+
1.91 ;;; BlobDB
2.1--- a/lisp/ffi/rocksdb/opts.lisp Sun Dec 10 23:02:43 2023 -0500
2.2+++ b/lisp/ffi/rocksdb/opts.lisp Mon Dec 11 22:05:25 2023 -0500
2.3@@ -1,12 +1,150 @@
2.4 (in-package :rocksdb)
2.5
2.6+(defvar *rocksdb-compression-backends* '(snappy zlib bz2 lz4 lz4hc xpress zstd))
2.7+
2.8+(defvar *rocksdb-compaction-levels* '(level universal fifo))
2.9+
2.10+(defvar *rocksdb-perf-metrics*
2.11+ '(user-key-comparison-count block-cache-hit-count
2.12+ block-read-count block-read-byte
2.13+ block-read-time block-checksum-time
2.14+ block-decompress-time get-read-bytes
2.15+ multiget-read-bytes iter-read-bytes
2.16+ internal-key-skipped-count internal-delete-skipped-count
2.17+ internal-recent-skipped-count internal-merge-count
2.18+ get-snapshot-time get-from-memtable-time
2.19+ get-from-memtable-count get-post-process-time
2.20+ get-from-output-files-time seek-on-memtable-time
2.21+ seek-on-memtable-count next-on-memtable-count
2.22+ prev-on-memtable-count seek-child-seek-time
2.23+ seek-child-seek-count seek-min-heap-time
2.24+ seek-max-heap-time seek-internal-seek-time
2.25+ find-next-user-entry-time write-wal-time
2.26+ write-memtable-time write-delay-time
2.27+ write-pre-and-post-process-time db-mutex-lock-nanos
2.28+ db-condition-wait-nanos merge-operator-time-nanos
2.29+ read-index-block-nanos read-filter-block-nanos
2.30+ new-table-block-iter-nanos new-table-iterator-nanos
2.31+ block-seek-nanos find-table-nanos
2.32+ bloom-memtable-hit-count bloom-memtable-miss-count
2.33+ bloom-sst-hit-count bloom-sst-miss-count
2.34+ key-lock-wait-time key-lock-wait-count
2.35+ env-new-sequential-file-nanos env-new-random-access-file-nanos
2.36+ env-new-writable-file-nanos env-reuse-writable-file-nanos
2.37+ env-new-random-rw-file-nanos env-new-directory-nanos
2.38+ env-file-exists-nanos env-get-children-nanos
2.39+ env-get-children-file-attributes-nanos env-delete-file-nanos
2.40+ env-create-dir-nanos env-create-dir-if-missing-nanos
2.41+ env-delete-dir-nanos env-get-file-size-nanos
2.42+ env-get-file-modification-time-nanos env-rename-file-nanos
2.43+ env-link-file-nanos env-lock-file-nanos
2.44+ env-unlock-file-nanos env-new-logger-nanos
2.45+ number-async-seek blob-cache-hit-count
2.46+ blob-read-count blob-read-byte
2.47+ blob-read-time blob-checksum-time
2.48+ blob-decompress-time internal-range-del-reseek-count
2.49+ block-read-cpu-time total-metric-count))
2.50+
2.51+(defvar *rocksdb-statistics-levels*
2.52+ (map 'vector
2.53+ (lambda (x) (string-downcase (symbol-name x)))
2.54+ '(disable-all except-tickers except-histogram-or-timers
2.55+ except-timers except-detailed-timers except-time-for-mutex
2.56+ all)))
2.57+
2.58+(defvar *rocksdb-options*
2.59+ (map 'vector
2.60+ (lambda (x) (string-downcase (symbol-name x)))
2.61+ '(create-if-missing create-missing-column-families error-if-exists
2.62+ paranoid-checks info-log-level write-buffer-size db-write-buffer-size
2.63+ max-open-files max-file-opening-threads max-total-wal-size compression-options
2.64+ compression-options-zstd-max-train-bytes compression-options-max-dict-buffer-bytes
2.65+ compression-options-parallel-threads compression-options-use-zstd-dict-trainer
2.66+ num-levels level0-file-num-compaction-trigger level0-slowdown-writes-trigger
2.67+ level0-stop-writes-trigger target-file-size-base target-file-size-multiplier
2.68+ max-bytes-for-level-base level-compaction-dynamic-level-bytes max-bytes-for-level-multiplier
2.69+ block-based-table-factory allow-ingest-behind merge-operator statistics-level
2.70+ skip-stats-update-on-db-open skip-checking-sst-filie-sizes-on-db-open enable-blob-files
2.71+ min-blob-size blob-file-size blob-compression-type enable-blob-gc blob-gc-age-cutoff
2.72+ blob-gc-force-threshold blob-compaction-readahead-size blob-file-starting-level
2.73+ prepopulate-blob-cache))
2.74+ "Provides early list of options for macros to populate.")
2.75+
2.76+(define-opt rocksdb-ingestexternalfileoptions)
2.77+(define-opt rocksdb-backup-engine-options)
2.78+(define-opt rocksdb-restore-options)
2.79+(define-opt rocksdb-hyper-clock-cache-options)
2.80+(define-opt rocksdb-fifo-compaction-options)
2.81+(define-opt rocksdb-transactiondb-options)
2.82+(define-opt rocksdb-transaction-options)
2.83+(define-opt rocksdb-optimistictransaction-options)
2.84+(define-opt rocksdb-envoptions)
2.85+(define-opt rocksdb-universal-compaction-options)
2.86+
2.87+(define-opaque rocksdb-wal-readoptions)
2.88+
2.89 (define-opaque rocksdb-block-based-table-options)
2.90+(define-alien-routine rocksdb-block-based-options-create (* rocksdb-block-based-table-options))
2.91+(define-alien-routine rocksdb-block-based-options-destroy void (* rocksdb-block-based-table-options))
2.92+
2.93+(define-alien-routine rocksdb-block-based-options-set-checksum void
2.94+ (opt (* rocksdb-block-based-table-options)) (val char))
2.95+
2.96+(define-alien-routine rocksdb-block-based-options-set-block-size void
2.97+ (opt (* rocksdb-block-based-table-options)) (block-size size-t))
2.98+
2.99+(define-alien-routine rocksdb-block-based-options-set-block-size-deviation void
2.100+ (opt (* rocksdb-block-based-table-options)) (block-size-deviation int))
2.101+
2.102+(define-alien-routine rocksdb-block-based-options-set-block-restart-interval void
2.103+ (opt (* rocksdb-block-based-table-options)) (block-restart-interval int))
2.104+
2.105+(define-alien-routine rocksdb-block-based-options-set-index-block-restart-interval void
2.106+ (opt (* rocksdb-block-based-table-options)) (index-block-restart-interval char))
2.107+
2.108+(define-alien-routine rocksdb-block-based-options-set-metadata-block-size void
2.109+ (opt (* rocksdb-block-based-table-options)) (metadata-block-size unsigned-long))
2.110+
2.111+(define-alien-routine rocksdb-block-based-options-set-partition-filters void
2.112+ (opt (* rocksdb-block-based-table-options)) (partition-filters unsigned-char))
2.113+
2.114+(define-alien-routine rocksdb-block-based-options-set-partition-filters-for-memory void
2.115+ (opt (* rocksdb-block-based-table-options)) (optimize-filters-for-memory unsigned-char))
2.116+
2.117+(define-alien-routine rocksdb-block-based-options-set-use-delta-encoding void
2.118+ (opt (* rocksdb-block-based-table-options)) (use-delta-encoding unsigned-char))
2.119+
2.120+(define-alien-routine rocksdb-block-based-options-set-no-block-cache void
2.121+ (opt (* rocksdb-block-based-table-options)) (no-block-cache unsigned-char))
2.122
2.123 (define-alien-routine rocksdb-block-based-options-set-block-cache void
2.124 (opt (* rocksdb-block-based-table-options)) (block-cache (* rocksdb-cache)))
2.125
2.126+(define-alien-routine rocksdb-block-based-options-set-format-version void
2.127+ (opt (* rocksdb-block-based-table-options)) (val int))
2.128+
2.129+(define-alien-routine rocksdb-block-based-options-set-index-type void
2.130+ (opt (* rocksdb-block-based-table-options)) (val int))
2.131+
2.132+(define-alien-routine rocksdb-block-based-options-set-data-block-index-type void
2.133+ (opt (* rocksdb-block-based-table-options)) (val int))
2.134+
2.135+(define-alien-routine rocksdb-block-based-options-set-data-block-hash-ratio void
2.136+ (opt (* rocksdb-block-based-table-options)) (val double))
2.137+
2.138 (define-alien-routine rocksdb-block-based-options-set-cache-index-and-filter-blocks void
2.139- (opt (* rocksdb-block-based-table-options)) (cache-index-and-filter-blocks c-string))
2.140+ (opt (* rocksdb-block-based-table-options)) (val unsigned-char))
2.141+
2.142+(define-alien-routine rocksdb-block-based-options-set-cache-index-and-filter-blocks-with-high-priority void
2.143+ (opt (* rocksdb-block-based-table-options)) (val unsigned-char))
2.144+
2.145+(define-alien-routine rocksdb-block-based-options-set-pin-l0-filter-and-index-blocks-in-cache void
2.146+ (opt (* rocksdb-block-based-table-options)) (val unsigned-char))
2.147+
2.148+(define-alien-routine rocksdb-block-based-options-set-pin-top-level-index-and-filter void
2.149+ (opt (* rocksdb-block-based-table-options)) (val unsigned-char))
2.150+
2.151+(define-opaque rocksdb-cuckoo-table-options)
2.152
2.153 (define-opt rocksdb-options
2.154 create-if-missing
2.155@@ -36,7 +174,8 @@
2.156 (block-based-table-factory
2.157 (table-options (* rocksdb-block-based-table-options)))
2.158 (allow-ingest-behind (val unsigned-char))
2.159- (merge-operator (comparator (* rocksdb-comparator)))
2.160+ (comparator (val (* rocksdb-comparator)))
2.161+ (merge-operator (val (* rocksdb-mergeoperator)))
2.162 (statistics-level (level int))
2.163 (skip-stats-update-on-db-open (val unsigned-char))
2.164 (skip-checking-sst-filie-sizes-on-db-open (val unsigned-char))
2.165@@ -49,29 +188,49 @@
2.166 (blob-gc-force-threshold (val double))
2.167 (blob-compaction-readahead-size (val unsigned-long))
2.168 (blob-file-starting-level (val int))
2.169+ (blob-cache (val (* rocksdb-cache)))
2.170 (prepopulate-blob-cache (val int)))
2.171
2.172+(define-alien-routine rocksdb-options-increase-parallelism void
2.173+ (opt (* rocksdb-options)) (total-threads int))
2.174+
2.175+(define-alien-routine rocksdb-options-optimize-level-style-compaction void
2.176+ (opt (* rocksdb-options))
2.177+ (memtable-memory-budget unsigned-long))
2.178+
2.179 (define-alien-routine rocksdb-options-enable-statistics void
2.180 (* rocksdb-options))
2.181+
2.182 ;; (define-alien-routine rocksdb-options-set-db-paths void
2.183 ;; (opt (* rocksdb-options)))
2.184
2.185-(define-alien-routine rocksdb-options-set-blob-cache void
2.186- (opt (* rocksdb-options)) (blob-cache (* rocksdb-cache)))
2.187+;; (define-alien-routine rocksdb-options-set-uint64add-merge-operator void
2.188+;; (opt (* rocksdb-options)))
2.189
2.190 (define-opt rocksdb-writeoptions)
2.191 (define-opt rocksdb-readoptions)
2.192 (define-opt rocksdb-flushoptions
2.193 (wait (val unsigned-char)))
2.194
2.195-(define-alien-routine rocksdb-options-increase-parallelism void
2.196- (opt (* rocksdb-options)) (total-threads int))
2.197+(define-opt rocksdb-compactoptions
2.198+ (exclusive-manual-compaction (val unsigned-char))
2.199+ (bottommost-level-compaction (val unsigned-char))
2.200+ (change-level (val unsigned-char))
2.201+ (target-level (val int)))
2.202+
2.203+(define-opt rocksdb-lru-cache-options)
2.204
2.205-(define-alien-routine rocksdb-options-optimize-level-style-compaction void
2.206- (opt (* rocksdb-options))
2.207- (memtable-memory-budget unsigned-long))
2.208+(define-alien-routine rocksdb-lru-cache-options-set-capacity void
2.209+ (self (* rocksdb-lru-cache-options))
2.210+ (val size-t))
2.211
2.212-(define-alien-routine rocksdb-flushoptions-get-wait unsigned-char (* rocksdb-flushoptions))
2.213+(define-alien-routine rocksdb-lru-cache-options-set-num-shard-bits void
2.214+ (self (* rocksdb-lru-cache-options))
2.215+ (val int))
2.216+
2.217+(define-alien-routine rocksdb-lru-cache-options-set-memory-allocator void
2.218+ (self (* rocksdb-lru-cache-options))
2.219+ (val (* rocksdb-memory-allocator)))
2.220
2.221 (def-with-errptr rocksdb-load-latest-options
2.222 void
2.223@@ -91,7 +250,7 @@
2.224 (len size-t))
2.225
2.226 (def-with-errptr
2.227- rocksdb-set-options
2.228+ rocksdb-set-options
2.229 void
2.230 (db (* rocksdb))
2.231 (count int)
2.232@@ -105,3 +264,6 @@
2.233 (count int)
2.234 (keys (array c-string))
2.235 (values (array c-string)))
2.236+
2.237+(define-alien-routine rocksdb-options-create-copy (* rocksdb-options)
2.238+ (src (* rocksdb-options)))
3.1--- a/lisp/ffi/rocksdb/pkg.lisp Sun Dec 10 23:02:43 2023 -0500
3.2+++ b/lisp/ffi/rocksdb/pkg.lisp Mon Dec 11 22:05:25 2023 -0500
3.3@@ -146,6 +146,12 @@
3.4 ;; CACHE
3.5 :rocksdb-cache
3.6 :rocksdb-cache-create-lru
3.7+ :rocksdb-lru-cache-options
3.8+ :rocksdb-lru-cache-options-create
3.9+ :rocksdb-lru-cache-options-destroy
3.10+ :rocksdb-lru-cache-options-set-capacity
3.11+ :rocksdb-lru-cache-options-set-num-shard-bits
3.12+ :rocksdb-lru-cache-options-set-memory-allocator
3.13 ;; BLOCK-BASED OPTIONS
3.14 :rocksdb-block-based-table-options
3.15 :rocksdb-block-based-options-create
3.16@@ -153,6 +159,7 @@
3.17 :rocksdb-block-based-options-set-block-cache
3.18 :rocksdb-block-based-options-set-cache-index-and-filter-blocks
3.19 ;; OPTIONS
3.20+ :*rocksdb-options*
3.21 ;; opt-utils
3.22 :rocksdb-load-latest-options
3.23 :rocksdb-load-latest-options-destroy
3.24@@ -160,6 +167,7 @@
3.25 :rocksdb-set-options-cf
3.26 :rocksdb-options
3.27 :rocksdb-options-create
3.28+ :rocksdb-options-create-copy
3.29 :rocksdb-options-destroy
3.30 :rocksdb-options-increase-parallelism
3.31 :rocksdb-options-optimize-for-point-lookup
3.32@@ -308,6 +316,33 @@
3.33 :rocksdb-approximate-memory-usage-get-mem-table-unflushed
3.34 :rocksdb-approximate-memory-usage-get-mem-table-readers-total
3.35 :rocksdb-approximate-memory-usage-get-cache-total
3.36+ ;; perfcontext
3.37+ :rocksdb-set-perf-level
3.38+ :rocksdb-perfcontext-create
3.39+ :rocksdb-perfcontext-reset
3.40+ :rocksdb-perfcontext-report
3.41+ :rocksdb-perfcontext-metric
3.42+ :rocksdb-perfcontext-destroy
3.43+ ;; compactionfilter
3.44+ :rocksdb-compactionfilter-create
3.45+ :rocksdb-compactionfilter-set-ignore-snapshots
3.46+ :rocksdb-compactionfilter-destroy
3.47+ ;; compactionfiltercontext
3.48+ :rocksdb-compactionfiltercontext-is-full-compaction
3.49+ :rocksdb-compactionfiltercontext-is-manual-compaction
3.50+ ;; comparator
3.51+ :rocksdb-comparator-create
3.52+ :rocksdb-comparator-destroy
3.53+ :rocksdb-comparator-with-ts-create
3.54+ ;; filterpolicy
3.55+ :rocksdb-filterpolicy-destroy
3.56+ :rocksdb-filterpolicy-create-bloom
3.57+ :rocksdb-filterpolicy-create-bloom-full
3.58+ :rocksdb-filterpolicy-create-ribbon
3.59+ :rocksdb-filterpolicy-create-ribbon-hybrid
3.60+ ;; mergeoperator
3.61+ :rocksdb-mergeoperator-create
3.62+ :rocksdb-mergeoperator-destroy
3.63 ;; stats
3.64 :rocksdb-statistics-histogram-data
3.65 :rocksdb-statistics-histogram-data-create
3.66@@ -328,3 +363,5 @@
3.67 (unless (member :rocksdb *features*)
3.68 (sb-alien:load-shared-object "librocksdb.so" :dont-save t)
3.69 (push :rocksdb *features*)))
3.70+
3.71+(load-rocksdb)
4.1--- a/lisp/ffi/rocksdb/types.lisp Sun Dec 10 23:02:43 2023 -0500
4.2+++ b/lisp/ffi/rocksdb/types.lisp Mon Dec 11 22:05:25 2023 -0500
4.3@@ -3,15 +3,10 @@
4.4 (define-alien-type rocksdb-errptr (* (* t)))
4.5
4.6 (define-opaque rocksdb)
4.7-(define-opaque rocksdb)
4.8 (define-opaque rocksdb-iterator)
4.9 (define-opaque rocksdb-backup-engine)
4.10 (define-opaque rocksdb-backup-engine-info)
4.11-(define-opaque rocksdb-backup-engine-options)
4.12-(define-opaque rocksdb-restore-options)
4.13 (define-opaque rocksdb-memory-allocator)
4.14-(define-opaque rocksdb-lru-cache-options)
4.15-(define-opaque rocksdb-hyper-clock-cache-options)
4.16 (define-opaque rocksdb-cache)
4.17 (define-opaque rocksdb-compactionfilter)
4.18 (define-opaque rocksdb-compactionfiltercontext)
4.19@@ -19,13 +14,10 @@
4.20 (define-opaque rocksdb-comparator)
4.21 (define-opaque rocksdb-dbpath)
4.22 (define-opaque rocksdb-env)
4.23-(define-opaque rocksdb-fifo-compaction-options)
4.24 (define-opaque rocksdb-filelock)
4.25 (define-opaque rocksdb-filterpolicy)
4.26 (define-opaque rocksdb-logger)
4.27 (define-opaque rocksdb-mergeoperator)
4.28-(define-opaque rocksdb-compactoptions)
4.29-(define-opaque rocksdb-cuckoo-table-options)
4.30 (define-opaque rocksdb-randomfile)
4.31 (define-opaque rocksdb-seqfile)
4.32 (define-opaque rocksdb-slicetransform)
4.33@@ -37,22 +29,15 @@
4.34 (define-opaque rocksdb-column-family-metadata)
4.35 (define-opaque rocksdb-level-metadata)
4.36 (define-opaque rocksdb-sst-file-metadata)
4.37-(define-opaque rocksdb-envoptions)
4.38-(define-opaque rocksdb-ingestexternalfileoptions)
4.39 (define-opaque rocksdb-sstfilewriter)
4.40 (define-opaque rocksdb-ratelimiter)
4.41 (define-opaque rocksdb-perfcontext)
4.42 (define-opaque rocksdb-pinnableslice)
4.43-(define-opaque rocksdb-transactiondb-options)
4.44 (define-opaque rocksdb-transactiondb)
4.45-(define-opaque rocksdb-transaction-options)
4.46 (define-opaque rocksdb-optimistictransactiondb)
4.47-(define-opaque rocksdb-optimistictransaction-options)
4.48 (define-opaque rocksdb-transaction)
4.49 (define-opaque rocksdb-checkpoint)
4.50 (define-opaque rocksdb-wal-iterator)
4.51-(define-opaque rocksdb-wal-readoptions)
4.52 (define-opaque rocksdb-memory-comsumers)
4.53 (define-opaque rocksdb-memory-usage)
4.54-(define-opaque rocksdb-universal-compaction-options)
4.55 (define-opaque rocksdb-statistics-histogram-data)
5.1--- a/lisp/lib/rdb/obj.lisp Sun Dec 10 23:02:43 2023 -0500
5.2+++ b/lisp/lib/rdb/obj.lisp Mon Dec 11 22:05:25 2023 -0500
5.3@@ -1,22 +1,121 @@
5.4 (in-package :rdb)
5.5
5.6 ;;; rdb-opts
5.7-(defstruct rdb-opts
5.8- (create-if-missing nil :type boolean)
5.9- (total-threads 1 :type integer) ;; numcpus is default
5.10- (max-open-files 10000 :type integer)
5.11- (use-fsync nil :type boolean)
5.12- (disable-auto-compactions nil :type boolean)
5.13- (sap nil :type (or null alien)))
5.14+(defvar *rdb-opts-lookup-table*
5.15+ (let ((table (make-hash-table :test #'equal)))
5.16+ (mapc (lambda (x) (setf (gethash (car x) table) (cdr x)))
5.17+ (loop for y across *rocksdb-options*
5.18+ collect (cons y (format nil "rocksdb-options-set-~x" y))))
5.19+ table))
5.20+
5.21+(defun rdb-opt-setter (key)
5.22+ (find-symbol (format nil "rocksdb-options-set-~x" key) :rocksdb))
5.23+
5.24+(defun %set-rocksdb-option (opt key val)
5.25+ (funcall (rdb-opt-setter key) opt val))
5.26+
5.27+;; (funcall (rdb-opt-setter "create-if-missing") (rocksdb-options-create) nil)
5.28+
5.29+(defclass rdb-opts ()
5.30+ ((table :initform (make-hash-table :test #'equal) :type hash-table :accessor rdb-opts-table)
5.31+ (sap :type (or null alien) :accessor rdb-opts-sap)))
5.32
5.33+(defmethod initialize-instance :after ((self rdb-opts) &rest initargs &key &allow-other-keys)
5.34+ (with-slots (sap) self
5.35+ (unless (getf initargs :sap) (setf sap (rocksdb-options-create)))
5.36+ (loop for (k v) on initargs by #'cddr while v
5.37+ do (let ((k (typecase k
5.38+ (string (string-downcase k))
5.39+ (symbol (string-downcase (symbol-name k)))
5.40+ (t (string-downcase (format nil "~x" k))))))
5.41+ (set-opt self k v)))
5.42+ self))
5.43+
5.44+(defmethod get-opt ((self rdb-opts) key)
5.45+ "Return the current value of KEY in SELF if found, else return nil."
5.46+ (gethash key (rdb-opts-table self)))
5.47+
5.48+(defmethod set-opt ((self rdb-opts) key val &key push)
5.49+ "Set the VAL of KEY in SELF with '(setf (gethash SELF KEY) VAL)'."
5.50+ (prog1
5.51+ (setf (gethash key (rdb-opts-table self)) val)
5.52+ (when push (push-sap self key))))
5.53+
5.54+(defmethod push-sap ((self rdb-opts) key)
5.55+ "Push KEY from slot :TABLE to the instance :SAP."
5.56+ (%set-rocksdb-option (rdb-opts-sap self) key (get-opt self key)))
5.57+
5.58+(defmethod push-sap* ((self rdb-opts))
5.59+ "Initialized the SAP slot with values from TABLE."
5.60+ (with-slots (table) self
5.61+ (loop for k across (hash-table-keys table)
5.62+ do (push-sap self k))))
5.63+
5.64+(declaim (inline default-rdb-opts))
5.65 (defun default-rdb-opts ()
5.66- (make-rdb-opts
5.67- :create-if-missing t
5.68- :total-threads 4))
5.69+ (make-instance 'rdb-opts
5.70+ :create-if-missing t
5.71+ :total-threads 4
5.72+ :max-open-files 10000))
5.73
5.74 ;;; rdb-cf
5.75+
5.76+;; NOTE: read-sequence and write-sequence now accept user-extended
5.77+;; sequences
5.78+
5.79+(defclass rdb-bytes (sequence)
5.80+ ((buffer :initarg :buffer :type (array unsigned-byte) :accessor rdb-bytes-buffer))
5.81+ (:documentation "RDB unsigned-byte array. Implements the iterator protocol."))
5.82+
5.83+(defmethod sequence:length ((self rdb-bytes))
5.84+ (length (rdb-bytes-buffer self)))
5.85+
5.86+(defmethod sequence:elt ((self rdb-bytes) index)
5.87+ (elt (rdb-bytes-buffer self) index))
5.88+
5.89+(defmethod sequence:make-sequence-like ((self rdb-bytes) length &key initial-element initial-contents)
5.90+ (let ((res (make-instance 'rdb-bytes)))
5.91+ (cond
5.92+ ((and initial-element initial-contents) (error "supplied both ~S and ~S to ~S" :initial-element :initial-contents 'make-sequence-like))
5.93+ (initial-element (setf (rdb-bytes-buffer res) (make-array length :element-type (array-element-type self)
5.94+ :initial-element initial-element)))
5.95+ (initial-contents (setf (rdb-bytes-buffer res) (make-array length :element-type (array-element-type self)
5.96+ :initial-contents initial-contents)))
5.97+ (t (setf (rdb-bytes-buffer res) (make-array length :element-type (array-element-type self)))))))
5.98+
5.99+;; (sequence:make-sequence-iterator (make-instance 'rdb-bytes :buffer (vector 1 2 3)))
5.100+(defmethod sequence:make-sequence-iterator ((self rdb-bytes) &key from-end start end)
5.101+ (sequence:make-sequence-iterator (rdb-bytes-buffer self) :from-end from-end :start start :end end))
5.102+
5.103+;; (defmethod sequence:subseq ((self rdb-bytes) start &optional end))
5.104+;; (defmethod sequence:concatenate ((self rdb-bytes) &rest sequences))
5.105+
5.106+(defclass rdb-val (rdb-bytes)
5.107+ ()
5.108+ (:documentation "RDB value protocol.
5.109+
5.110+Values must be able to be encoded to and from (array unsigned-byte)."))
5.111+
5.112+(defclass rdb-key (rdb-bytes)
5.113+ ()
5.114+ (:documentation "RDB key protocol.
5.115+
5.116+Keys must be able to be encoded to and from (array unsigned-byte)."))
5.117+
5.118+(defclass rdb-kv (rdb-bytes)
5.119+ ((key :initarg :key :type rdb-key)
5.120+ (val :initarg :val :type rdb-val)))
5.121+
5.122+(defun make-rdb-key (key)
5.123+ "Convert KEY to an object of type RDB-KEY."
5.124+ (make-instance 'rdb-key :buffer key))
5.125+
5.126 (defstruct rdb-cf
5.127+ "RDB Column Family structure. Contains a name, a cons of (rdb-key-type
5.128+. rdb-val-type), and a system-area-pointer to the underlying
5.129+rocksdb_cf_t handle."
5.130 (name "" :type string)
5.131+ (kv (make-instance 'rdb-kv) :type rdb-kv)
5.132 (sap nil :type (or null alien)))
5.133
5.134 (defun create-cf (db cf)
5.135@@ -34,23 +133,23 @@
5.136 (defmethod push-cf ((cf rdb-cf) (db rdb))
5.137 (vector-push cf (rdb-cfs db)))
5.138
5.139-(defmethod open-rdb ((self rdb))
5.140+(defmethod open-db ((self rdb))
5.141 (setf (rdb-db self)
5.142- (open-db-raw (rdb-name self) (rdb-opts self))))
5.143+ (open-db-raw (rdb-name self) (rdb-opts-sap (rdb-opts self)))))
5.144
5.145-(defmethod close-rdb ((self rdb))
5.146+(defmethod close-db ((self rdb))
5.147 (close-db-raw (rdb-db self))
5.148 (setf (rdb-db self) nil))
5.149
5.150-(defmethod destroy-rdb ((self rdb))
5.151+(defmethod destroy-db ((self rdb))
5.152 (when (rdb-db self) (close-rdb self))
5.153 (destroy-db-raw (rdb-name self)))
5.154
5.155-(defmethod init-cfs ((self rdb))
5.156+(defmethod init-db ((self rdb))
5.157 (loop for cf across (rdb-cfs self)
5.158 do (create-cf (rdb-db self) cf)))
5.159
5.160-(defmethod insert-kv ((self rdb) key val &key cf)
5.161+(defmethod insert-key ((self rdb) key val &key cf)
5.162 (if cf
5.163 (put-cf-raw
5.164 (rdb-db self)
5.165@@ -61,6 +160,3 @@
5.166 (rdb-db self)
5.167 key
5.168 val)))
5.169-
5.170-(defmethod insert-kv-str ((self rdb) key val &key cf)
5.171- (insert-kv self (string-to-octets key) (string-to-octets val) :cf cf))
6.1--- a/lisp/lib/rdb/pkg.lisp Sun Dec 10 23:02:43 2023 -0500
6.2+++ b/lisp/lib/rdb/pkg.lisp Mon Dec 11 22:05:25 2023 -0500
6.3@@ -26,15 +26,18 @@
6.4 :iter-key :iter-key-str
6.5 :iter-val :iter-val-str
6.6 ;; proto
6.7- :put-kv :put-kv-str
6.8- :get-kv :get-kv-str
6.9- :put-cf :put-cf-str
6.10- :get-cf :get-cf-str
6.11+ :put-key :put-kv
6.12+ :get-key :get-kv
6.13+ :put-cf-key :get-cf-key
6.14+ :push-sap :push-sap*
6.15+ :get-opt :set-opt
6.16+ :push-cf :init-db
6.17+ :insert-kv :insert-key
6.18+ :open-db :close-db :destroy-db
6.19 ;; obj
6.20- :rdb :make-rdb :open-rdb :close-rdb :destroy-rdb
6.21+ :rdb :make-rdb
6.22 :rdb-db :rdb-name :rdb-cfs :rdb-opts
6.23- :push-cf :init-cfs
6.24- :insert-kv :insert-kv-str
6.25+ :make-rdb-key :rdb-kv :rdb-key :rdb-val :rdb-bytes
6.26 :rdb-opts :make-rdb-opts
6.27 :default-rdb-opts
6.28 :rdb-cf :make-rdb-cf :create-cf
7.1--- a/lisp/lib/rdb/proto.lisp Sun Dec 10 23:02:43 2023 -0500
7.2+++ b/lisp/lib/rdb/proto.lisp Mon Dec 11 22:05:25 2023 -0500
7.3@@ -1,5 +1,20 @@
7.4 (in-package :rdb)
7.5
7.6-;; Most methods are intended to work with the RDB struct
7.7-(defgeneric put-kv (self key val &key &allow-other-keys))
7.8-(defgeneric get-kv (self key &key &allow-other-keys))
7.9+;; Most methods are intended to work with the RDB struct directly
7.10+(defgeneric put-kv (self kv))
7.11+(defgeneric put-key (self key val))
7.12+(defgeneric get-key (self key))
7.13+
7.14+(defgeneric get-opt (self key))
7.15+(defgeneric set-opt (self key val &key &allow-other-keys))
7.16+(defgeneric push-sap (self key))
7.17+(defgeneric push-sap* (self))
7.18+
7.19+(defgeneric push-cf (self cf))
7.20+
7.21+(defgeneric open-db (self))
7.22+(defgeneric close-db (self))
7.23+(defgeneric destroy-db (self))
7.24+(defgeneric init-db (self))
7.25+(defgeneric insert-key (self key val &key &allow-other-keys))
7.26+(defgeneric insert-kv (self kv &key &allow-other-keys))
8.1--- a/lisp/lib/rdb/tests.lisp Sun Dec 10 23:02:43 2023 -0500
8.2+++ b/lisp/lib/rdb/tests.lisp Mon Dec 11 22:05:25 2023 -0500
8.3@@ -7,12 +7,12 @@
8.4 (rocksdb:load-rocksdb)
8.5
8.6 (defmacro with-test-db-raw ((db-var path) &body body)
8.7- `(let ((,db-var (open-db-raw ,path)))
8.8+ `(let* ((opt (rdb::default-rocksdb-options))
8.9+ (,db-var (open-db-raw ,path opt)))
8.10 (unwind-protect (progn ,@body)
8.11- (rocksdb-close ,db-var)
8.12- (with-alien ((e rocksdb-errptr)
8.13- (o rocksdb-options (rocksdb-options-create)))
8.14- (rocksdb-destroy-db o ,path e)))))
8.15+ (with-errptr e
8.16+ (rocksdb-close ,db-var)
8.17+ (rocksdb-destroy-db opt ,path e)))))
8.18
8.19 (deftest rdb ()
8.20 "Test RDB struct and methods."
9.1--- a/lisp/lib/rdb/util.lisp Sun Dec 10 23:02:43 2023 -0500
9.2+++ b/lisp/lib/rdb/util.lisp Mon Dec 11 22:05:25 2023 -0500
9.3@@ -1,16 +1,16 @@
9.4 (in-package :rdb)
9.5
9.6 (defmacro with-errptr (e &body body)
9.7- `(with-alien ((,e rocksdb-errptr nil))
9.8- ,@body
9.9- (free-alien ,e)))
9.10+ `(with-alien ((,e rocksdb-errptr (make-alien rocksdb-errptr)))
9.11+ ,@body))
9.12+
9.13
9.14 (defun default-rocksdb-options ()
9.15 (let ((opts (rocksdb-options-create)))
9.16 (rocksdb-options-set-create-if-missing opts t)
9.17 opts))
9.18
9.19-(defun open-db-raw (db-path &optional (opts (default-rocksdb-options)))
9.20+(defun open-db-raw (db-path opts)
9.21 (with-errptr e
9.22 (let* ((db-path (if (pathnamep db-path)
9.23 (namestring db-path)
9.24@@ -32,7 +32,7 @@
9.25 (rocksdb-destroy-db opt path err))
9.26 (rocksdb-options-destroy opt)))
9.27
9.28-(defmacro with-open-db-raw ((db-var db-path &optional (opt (rocksdb-options-create))) &body body)
9.29+(defmacro with-open-db-raw ((db-var db-path &optional (opt (default-rocksdb-options))) &body body)
9.30 `(let ((,db-var (open-db-raw ,db-path ,opt)))
9.31 (unwind-protect (progn ,@body)
9.32 (rocksdb-close ,db-var)
10.1--- a/todo.org Sun Dec 10 23:02:43 2023 -0500
10.2+++ b/todo.org Mon Dec 11 22:05:25 2023 -0500
10.3@@ -103,3 +103,19 @@
10.4 - generated from RDB instances
10.5 - compiles in CF/option/multithread/type info
10.6 - clients implement custom Q language
10.7+** TODO sequence/iterator impl
10.8+- State "TODO" from [2023-12-10 Sun 23:09]
10.9+- use SBCL built-ins to extend the api - wrap the raw FFI iterator
10.10+- define mixins
10.11+** TODO comparator impl
10.12+- State "TODO" from [2023-12-10 Sun 23:11]
10.13+- supply custom comparators to rocksdb from lisp
10.14+** TODO type system
10.15+- State "TODO" from [2023-12-10 Sun 23:12]
10.16+- capture types of: kv, cf
10.17+- specs
10.18+ - kv-spec = (kv-spec val-spec)
10.19+ - must be serialized to/from vec<char>
10.20+ - cf-spec
10.21+ - used as superclass
10.22+ - methods for abstracting CFs
