1.1--- a/lisp/ffi/rocksdb/checkpoint.lisp Fri Aug 16 15:52:50 2024 -0400
1.2+++ b/lisp/ffi/rocksdb/checkpoint.lisp Fri Aug 16 18:06:48 2024 -0400
1.3@@ -16,7 +16,42 @@
1.4 (define-alien-routine rocksdb-checkpoint-object-destroy void
1.5 (* rocksdb-checkpoint))
1.6
1.7-;; rocksdb-open-and-trim-history
1.8+(def-with-errptr rocksdb-open-and-trim-history (* rocksdb)
1.9+ (opts (* rocksdb-options))
1.10+ (name c-string)
1.11+ (num-cfs int)
1.12+ (cf-names (array c-string))
1.13+ (cf-handles (array (* rocksdb-column-family-handle)))
1.14+ (trim-ts c-string)
1.15+ (trim-tslen size-t))
1.16+
1.17+(def-with-errptr rocksdb-open-column-families
1.18+ (* rocksdb)
1.19+ (options (* rocksdb-options))
1.20+ (name c-string)
1.21+ (num-column-families int)
1.22+ (column-family-names (array c-string))
1.23+ (column-family-options (array rocksdb-options))
1.24+ (column-family-handles (array rocksdb-column-family-handle)))
1.25+
1.26+(def-with-errptr rocksdb-open-column-families-with-ttl (* rocksdb)
1.27+ (opts (* rocksdb-options))
1.28+ (name c-string)
1.29+ (num-cfs int)
1.30+ (cf-names (array c-string))
1.31+ (cf-opts (array (* rocksdb-options)))
1.32+ (cf-handles (array (* rocksdb-column-family-handle)))
1.33+ (ttls (array int)))
1.34+
1.35+(def-with-errptr rocksdb-open-for-read-only-column-families (* rocksdb)
1.36+ (opts (* rocksdb-options))
1.37+ (name c-string)
1.38+ (num-cfs int)
1.39+ (cf-names (array c-string))
1.40+ (cf-opts (array (* rocksdb-options)))
1.41+ (cf-handles (array (* rocksdb-column-family-handle)))
1.42+ (err-if-wal-exists unsigned-char))
1.43+
1.44 (def-with-errptr rocksdb-open-as-secondary-column-families (* rocksdb)
1.45 (opts (* rocksdb-options))
1.46 (name c-string)
2.1--- a/lisp/ffi/rocksdb/compaction.lisp Fri Aug 16 15:52:50 2024 -0400
2.2+++ b/lisp/ffi/rocksdb/compaction.lisp Fri Aug 16 18:06:48 2024 -0400
2.3@@ -41,8 +41,8 @@
2.4 ;;; Compaction Filter Factory
2.5 (define-alien-routine rocksdb-compactionfilter-create (* rocksdb-compactionfilter)
2.6 (state (* t))
2.7- (destructor (* t))
2.8- (create-compaction-filter (* unsigned-char))
2.9+ (destructor (* rocksdb-destructor-function))
2.10+ (generator (* rocksdb-create-compaction-filter-function))
2.11 (context (* rocksdb-compactionfiltercontext)))
2.12
2.13 (define-alien-routine rocksdb-compacitonfilter-destroy void
2.14@@ -58,3 +58,4 @@
2.15 ((state (* t))
2.16 (context (* rocksdb-compactionfiltercontext)))
2.17 (declare (ignore state context)))
2.18+
3.1--- a/lisp/ffi/rocksdb/comparator.lisp Fri Aug 16 15:52:50 2024 -0400
3.2+++ b/lisp/ffi/rocksdb/comparator.lisp Fri Aug 16 18:06:48 2024 -0400
3.3@@ -25,7 +25,7 @@
3.4 c-string
3.5 size-t))
3.6
3.7-(define-alien-type rocksdb-compare-ts-function
3.8+(define-alien-type rocksdb-compare-with-ts-function
3.9 (function int
3.10 (* t)
3.11 c-string
3.12@@ -45,9 +45,9 @@
3.13
3.14 (define-alien-routine rocksdb-comparator-create (* rocksdb-comparator)
3.15 (state (* t))
3.16- (destructor (* t))
3.17- (compare (* int))
3.18- (name (* unsigned-char)))
3.19+ (destructor (* rocksdb-destructor-function))
3.20+ (compare (* rocksdb-compare-function))
3.21+ (name (* rocksdb-name-function)))
3.22
3.23 ;; (rocksdb-comparator-create nil nil (make-alien int 1) (make-alien unsigned-char 10))
3.24
3.25@@ -55,8 +55,41 @@
3.26
3.27 (define-alien-routine rocksdb-comparator-with-ts-create (* rocksdb-comparator)
3.28 (state (* t))
3.29- (destructor (* t))
3.30- (compare (* int))
3.31- (compare-ts (* int))
3.32- (compare-without-ts (* int))
3.33- (name (* unsigned-char)))
3.34+ (destructor (* rocksdb-destructor-function))
3.35+ (compare (* rocksdb-compare-function))
3.36+ (compare-with-ts (* rocksdb-compare-with-ts-function))
3.37+ (compare-without-ts (* rocksdb-compare-without-ts-function))
3.38+ (name (* rocksdb-name-function)))
3.39+
3.40+(define-alien-callable rocksdb-compare-never-name c-string () (make-alien-string "compare-never"))
3.41+
3.42+(define-alien-callable rocksdb-compare-never int
3.43+ ((state (* t))
3.44+ (a c-string)
3.45+ (alen size-t)
3.46+ (b c-string)
3.47+ (blen size-t))
3.48+ (declare (ignore state a alen b blen))
3.49+ 0)
3.50+
3.51+(define-alien-callable rocksdb-compare-never-with-ts int
3.52+ ((state (* t))
3.53+ (a c-string)
3.54+ (alen size-t)
3.55+ (b c-string)
3.56+ (blen size-t))
3.57+ (declare (ignore state a alen b blen))
3.58+ 0)
3.59+
3.60+(define-alien-callable rocksdb-compare-never-without-ts int
3.61+ ((state (* t))
3.62+ (a c-string)
3.63+ (alen size-t)
3.64+ (a-ts unsigned-char)
3.65+ (b c-string)
3.66+ (blen size-t)
3.67+ (b-ts unsigned-char))
3.68+ (declare (ignore state a alen a-ts b blen b-ts))
3.69+ 0)
3.70+
3.71+
4.1--- a/lisp/ffi/rocksdb/db.lisp Fri Aug 16 15:52:50 2024 -0400
4.2+++ b/lisp/ffi/rocksdb/db.lisp Fri Aug 16 18:06:48 2024 -0400
4.3@@ -27,6 +27,11 @@
4.4 (val (* unsigned-char))
4.5 (vallen size-t))
4.6
4.7+(def-with-errptr rocksdb-write void
4.8+ (db (* rocksdb))
4.9+ (opts (* rocksdb-writeoptions))
4.10+ (batch (* rocksdb-writebatch)))
4.11+
4.12 (def-with-errptr rocksdb-get
4.13 (* unsigned-char)
4.14 (db (* rocksdb))
4.15@@ -95,6 +100,8 @@
4.16 (ts (* c-string))
4.17 (tslen (* size-t)))
4.18
4.19+(define-alien-routine rocksdb-get-db-identity c-string (db (* rocksdb)) (idlen (* size-t)))
4.20+
4.21 ;; NOTE 2023-12-19: only the VOID-returning functions in the multi-
4.22 ;; family perform parallel IO:
4.23 ;; https://github.com/facebook/rocksdb/wiki/MultiGet-Performance
4.24@@ -144,6 +151,40 @@
4.25 (ts-list-sizes (array size-t))
4.26 (errs (array rocksdb-errptr)))
4.27
4.28+(define-alien-routine rocksdb-batched-multi-get-cf void
4.29+ (db (* rocksdb))
4.30+ (opts (* rocksdb-readoptions))
4.31+ (cf (* rocksdb-column-family-handle))
4.32+ (nkeys size-t)
4.33+ (keys (array c-string))
4.34+ (key-sizes (array size-t))
4.35+ (values (array (* rocksdb-pinnableslice)))
4.36+ (errs (array (* rocksdb-errptr)))
4.37+ (sorted-input boolean))
4.38+
4.39+(define-alien-routine rocksdb-key-may-exist unsigned-char
4.40+ (db (* rocksdb))
4.41+ (opts (* rocksdb-readoptions))
4.42+ (key c-string)
4.43+ (key-len size-t)
4.44+ (value (* c-string))
4.45+ (val-len (* size-t))
4.46+ (timestamp c-string)
4.47+ (timestamp-len size-t)
4.48+ (value-found (* unsigned-char)))
4.49+
4.50+(define-alien-routine rocksdb-key-may-exist-cf unsigned-char
4.51+ (db (* rocksdb))
4.52+ (opts (* rocksdb-readoptions))
4.53+ (cf (* rocksdb-column-family-handle))
4.54+ (key c-string)
4.55+ (key-len size-t)
4.56+ (value (* c-string))
4.57+ (val-len (* size-t))
4.58+ (timestamp c-string)
4.59+ (timestamp-len size-t)
4.60+ (value-found (* unsigned-char)))
4.61+
4.62 (define-alien-routine rocksdb-cache-create-lru (* rocksdb-cache) (capacity size-t))
4.63
4.64 (def-with-errptr rocksdb-flush void
4.65@@ -213,6 +254,12 @@
4.66 (define-alien-routine rocksdb-create-column-families-destroy void
4.67 (list (array rocksdb-column-family-handle)))
4.68
4.69+(def-with-errptr rocksdb-create-column-family-with-ttl (* rocksdb-column-family-handle)
4.70+ (db (* rocksdb))
4.71+ (cf-opts (* rocksdb-options))
4.72+ (cf-name c-string)
4.73+ (ttl int))
4.74+
4.75 (define-alien-routine rocksdb-column-family-handle-destroy void
4.76 (cf (* rocksdb-column-family-handle)))
4.77
4.78@@ -228,14 +275,8 @@
4.79 (db (* rocksdb))
4.80 (handle (* rocksdb-column-family-handle)))
4.81
4.82-(def-with-errptr rocksdb-open-column-families
4.83- (* rocksdb)
4.84- (options (* rocksdb-options))
4.85- (name c-string)
4.86- (num-column-families int)
4.87- (column-family-names (array c-string))
4.88- (column-family-options (array rocksdb-options))
4.89- (column-family-handles (array rocksdb-column-family-handle)))
4.90+(define-alien-routine rocksdb-get-default-column-family-handle (* rocksdb-column-family-handle)
4.91+ (db (* rocksdb)))
4.92
4.93 (def-with-errptr rocksdb-list-column-families
4.94 (array c-string)
4.95@@ -257,6 +298,27 @@
4.96 (val (* unsigned-char))
4.97 (vallen size-t))
4.98
4.99+(def-with-errptr rocksdb-put-with-ts void
4.100+ (db (* rocksdb))
4.101+ (opt (* rocksdb-writeoptions))
4.102+ (key (* unsigned-char))
4.103+ (keylen size-t)
4.104+ (ts c-string)
4.105+ (tslen size-t)
4.106+ (val (* unsigned-char))
4.107+ (vallen size-t))
4.108+
4.109+(def-with-errptr rocksdb-put-cf-with-ts void
4.110+ (db (* rocksdb))
4.111+ (opt (* rocksdb-writeoptions))
4.112+ (cf (* rocksdb-column-family-handle))
4.113+ (key (* unsigned-char))
4.114+ (keylen size-t)
4.115+ (ts c-string)
4.116+ (tslen size-t)
4.117+ (val (* unsigned-char))
4.118+ (vallen size-t))
4.119+
4.120 (def-with-errptr rocksdb-delete-cf
4.121 void
4.122 (db (* rocksdb))
4.123@@ -265,6 +327,66 @@
4.124 (key (* unsigned-char))
4.125 (keylen size-t))
4.126
4.127+(def-with-errptr rocksdb-delete-with-ts
4.128+ void
4.129+ (db (* rocksdb))
4.130+ (options (* rocksdb-writeoptions))
4.131+ (key (* unsigned-char))
4.132+ (ts c-string)
4.133+ (tslen size-t)
4.134+ (keylen size-t))
4.135+
4.136+(def-with-errptr rocksdb-delete-cf-with-ts
4.137+ void
4.138+ (db (* rocksdb))
4.139+ (options (* rocksdb-writeoptions))
4.140+ (cf (* rocksdb-column-family-handle))
4.141+ (key (* unsigned-char))
4.142+ (ts c-string)
4.143+ (tslen size-t)
4.144+ (keylen size-t))
4.145+
4.146+(def-with-errptr rocksdb-singledelete void
4.147+ (db (* rocksdb))
4.148+ (opts (* rocksdb-writeoptions))
4.149+ (key c-string)
4.150+ (keylen size-t))
4.151+
4.152+(def-with-errptr rocksdb-singledelete-with-ts void
4.153+ (db (* rocksdb))
4.154+ (opts (* rocksdb-writeoptions))
4.155+ (key c-string)
4.156+ (keylen size-t)
4.157+ (ts c-string)
4.158+ (tslen size-t))
4.159+
4.160+(def-with-errptr rocksdb-singledelete-cf-with-ts void
4.161+ (db (* rocksdb))
4.162+ (opts (* rocksdb-writeoptions))
4.163+ (cf (* rocksdb-column-family-handle))
4.164+ (key c-string)
4.165+ (keylen size-t)
4.166+ (ts c-string)
4.167+ (tslen size-t))
4.168+
4.169+(def-with-errptr rocksdb-singledelete-cf void
4.170+ (db (* rocksdb))
4.171+ (opts (* rocksdb-writeoptions))
4.172+ (cf (* rocksdb-column-family-handle))
4.173+ (key c-string)
4.174+ (keylen size-t))
4.175+
4.176+(def-with-errptr rocksdb-increase-full-history-ts-low void
4.177+ (db (* rocksdb))
4.178+ (cf (* rocksdb-column-family-handle))
4.179+ (ts-low c-string)
4.180+ (ts-lowlen size-t))
4.181+
4.182+(def-with-errptr rocksdb-get-full-history-ts-low c-string
4.183+ (db (* rocksdb))
4.184+ (cf (* rocksdb-column-family-handle))
4.185+ (ts-lowlen (* size-t)))
4.186+
4.187 (def-with-errptr rocksdb-delete-range-cf
4.188 void
4.189 (db (* rocksdb))
4.190@@ -340,6 +462,10 @@
4.191 (iter (* rocksdb-wal-iterator)))
4.192
4.193 ;;; Backup
4.194+(def-with-errptr rocksdb-backup-engine-verify-backup void
4.195+ (be (* rocksdb-backup-engine))
4.196+ (backup-id (unsigned 32)))
4.197+
4.198 (def-with-errptr rocksdb-backup-engine-open
4.199 (* rocksdb-backup-engine)
4.200 (opts (* rocksdb-options))
4.201@@ -368,6 +494,25 @@
4.202 (define-alien-routine rocksdb-backup-engine-close void
4.203 (be (* rocksdb-backup-engine)))
4.204
4.205+(define-alien-routine rocksdb-backup-engine-get-backup-info (* rocksdb-backup-engine-info)
4.206+ (be (* rocksdb-backup-engine)))
4.207+(define-alien-routine rocksdb-backup-engine-info-count int
4.208+ (info (* rocksdb-backup-engine-info)))
4.209+(define-alien-routine rocksdb-backup-engine-info-timestamp (signed 64)
4.210+ (info (* rocksdb-backup-engine-info))
4.211+ (index int))
4.212+(define-alien-routine rocksdb-backup-engine-info-backup-id (unsigned 64)
4.213+ (info (* rocksdb-backup-engine-info))
4.214+ (index int))
4.215+(define-alien-routine rocksdb-backup-engine-info-size (unsigned 64)
4.216+ (info (* rocksdb-backup-engine-info))
4.217+ (index int))
4.218+(define-alien-routine rocksdb-backup-engine-info-num-files (unsigned 32)
4.219+ (info (* rocksdb-backup-engine-info))
4.220+ (index int))
4.221+(define-alien-routine rocksdb-backup-engine-info-destroy void
4.222+ (info (* rocksdb-backup-engine-info)))
4.223+
4.224 ;;; Transactions
4.225 (def-with-errptr rocksdb-transactiondb-create-column-family (* rocksdb-column-family-handle)
4.226 (txn-db (* rocksdb-transactiondb))
5.1--- a/lisp/ffi/rocksdb/macs.lisp Fri Aug 16 15:52:50 2024 -0400
5.2+++ b/lisp/ffi/rocksdb/macs.lisp Fri Aug 16 18:06:48 2024 -0400
5.3@@ -49,4 +49,4 @@
5.4 ',(symbolicate opt '-set- n)) :rocksdb))))
5.5 `(progn ,@forms)))
5.6
5.7-;; (defmacro define-merge-operator-callbacks (name full partial operands)
5.8+(defmacro define-merge-operator-callbacks (name full partial state destructor delete-fn))
6.1--- a/lisp/ffi/rocksdb/merge.lisp Fri Aug 16 15:52:50 2024 -0400
6.2+++ b/lisp/ffi/rocksdb/merge.lisp Fri Aug 16 18:06:48 2024 -0400
6.3@@ -120,6 +120,8 @@
6.4
6.5 (define-alien-callable rocksdb-name c-string () (make-alien-string (symbol-name (gensym "rocksdb:"))))
6.6
6.7+(define-alien-callable rocksdb-concat-merge-name c-string () (make-alien-string "concat-merge"))
6.8+
6.9 (define-alien-callable rocksdb-concat-full-merge boolean
6.10 ((key (array unsigned-char))
6.11 (klen size-t)
7.1--- a/lisp/ffi/rocksdb/pkg.lisp Fri Aug 16 15:52:50 2024 -0400
7.2+++ b/lisp/ffi/rocksdb/pkg.lisp Fri Aug 16 18:06:48 2024 -0400
7.3@@ -249,7 +249,30 @@
7.4 :rocksdb-comparator-destroy
7.5 :rocksdb-comparator-create
7.6 :rocksdb-comparator-with-ts-create
7.7- :rocksdb-checkpoint-object-destroy))
7.8+ :rocksdb-checkpoint-object-destroy
7.9+ :rocksdb-name-function
7.10+ :rocksdb-concat-merge-name
7.11+ :rocksdb-compare-never-without-ts
7.12+ :rocksdb-compare-never-with-ts
7.13+ :rocksdb-compare-never-name
7.14+ :rocksdb-compare-never
7.15+ :rocksdb-compare-without-ts-function
7.16+ :rocksdb-compare-with-ts-function
7.17+ :rocksdb-compare-function
7.18+ :rocksdb-create-compaction-filter
7.19+ :rocksdb-create-compaction-filter-function
7.20+ :rocksdb-get-default-column-family-handle
7.21+ :rocksdb-get-db-identity
7.22+ :rocksdb-batched-multi-get-cf
7.23+ :rocksdb-key-may-exist
7.24+ :rocksdb-key-may-exist-cf
7.25+ :rocksdb-backup-engine-get-backup-info
7.26+ :rocksdb-backup-engine-info-count
7.27+ :rocksdb-backup-engine-info-timestamp
7.28+ :rocksdb-backup-engine-info-backup-id
7.29+ :rocksdb-backup-engine-info-size
7.30+ :rocksdb-backup-engine-info-num-files
7.31+ :rocksdb-backup-engine-info-destroy))
7.32
7.33 (in-package :rocksdb)
7.34
8.1--- a/lisp/ffi/rocksdb/tests.lisp Fri Aug 16 15:52:50 2024 -0400
8.2+++ b/lisp/ffi/rocksdb/tests.lisp Fri Aug 16 18:06:48 2024 -0400
8.3@@ -225,9 +225,6 @@
8.4 (delete-file file)
8.5 (is (null-alien errptr)))))
8.6
8.7-(deftest logger ()
8.8- "Test logging functionality.")
8.9-
8.10 (deftest stats ()
8.11 "Test statistics and performance-context related functionality."
8.12 (rocksdb-set-perf-level (rocksdb-perf-level "enable-time-except-for-mutex"))
8.13@@ -372,7 +369,7 @@
8.14 (is (zerop (parse-integer (rocksdb-property-value db (make-alien-string "rocksdb.num-files-at-level3")))))))
8.15
8.16 (deftest merge ()
8.17- "Test low-level merge-operator functionality using ALIEN-CALLBACKs."
8.18+ "Test low-level merge-operator functionality using Alien Callbacks."
8.19 (is (with-alien ((k (array unsigned-char))
8.20 (v (array unsigned-char))
8.21 (ops (array (array unsigned-char)))
8.22@@ -397,8 +394,43 @@
8.23 (string-trim "rocksdb:" (alien-funcall (alien-callable-function 'rocksdb-name))))))
8.24 ;; returns No Value
8.25 (is (null (alien-funcall (alien-callable-function 'rocksdb-concat-delete-value) nil "" 0)))
8.26- (is (null (alien-funcall (alien-callable-function 'rocksdb-destructor) (make-alien (* t))))))
8.27+ (is (null (alien-funcall (alien-callable-function 'rocksdb-destructor) (make-alien (* t)))))
8.28+
8.29+ ;; null merge op
8.30+ (with-alien ((state (* t))
8.31+ (destructor (* rocksdb-destructor-function))
8.32+ (full-merge (* rocksdb-full-merge-function))
8.33+ (partial-merge (* rocksdb-partial-merge-function))
8.34+ (delete-value (* rocksdb-delete-value-function))
8.35+ (name (* rocksdb-name-function)))
8.36+ (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name)
8.37+ '(alien (* rocksdb-mergeoperator)))))
8.38
8.39-(deftest comparator ())
8.40+ ;; concat merge op
8.41+ (with-alien ((state (* t))
8.42+ (destructor (* rocksdb-destructor-function) (alien-sap (alien-callable-function 'rocksdb-destructor)))
8.43+ (full-merge (* rocksdb-full-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-full-merge)))
8.44+ (partial-merge (* rocksdb-partial-merge-function) (alien-sap (alien-callable-function 'rocksdb-concat-partial-merge)))
8.45+ (delete-value (* rocksdb-delete-value-function) (alien-sap (alien-callable-function 'rocksdb-concat-delete-value)))
8.46+ (name (* rocksdb-name-function) (alien-sap (alien-callable-function 'rocksdb-concat-merge-name))))
8.47+ (is (typep (rocksdb-mergeoperator-create state destructor full-merge partial-merge delete-value name)
8.48+ '(alien (* rocksdb-mergeoperator))))))
8.49
8.50-(deftest compaction ())
8.51+(deftest comparator ()
8.52+ "Test low-level comparator API."
8.53+ (with-alien ((state (* t))
8.54+ (destructor (* rocksdb-destructor-function))
8.55+ (compare (* rocksdb-compare-function))
8.56+ (name (* rocksdb-name-function)))
8.57+ (rocksdb-comparator-create state destructor compare name)))
8.58+
8.59+(deftest compaction ()
8.60+ "Test low-level compactionfilter API."
8.61+ (with-alien ((state (* t))
8.62+ (context (* rocksdb-compactionfiltercontext))
8.63+ (destructor (* rocksdb-destructor-function))
8.64+ (generator (* rocksdb-create-compaction-filter-function)))
8.65+ (alien-funcall (alien-callable-function 'rocksdb-create-compaction-filter) state context)))
8.66+
8.67+(deftest logger ()
8.68+ "Test logging functionality.")
9.1--- a/lisp/lib/obj/color/util.lisp Fri Aug 16 15:52:50 2024 -0400
9.2+++ b/lisp/lib/obj/color/util.lisp Fri Aug 16 18:06:48 2024 -0400
9.3@@ -63,6 +63,9 @@
9.4 (unless (cl-ppcre:scan-to-strings comment-scanner line)
9.5 (multiple-value-bind (match registers)
9.6 (cl-ppcre:scan-to-strings color-scanner line)
9.7+ ;; we don't ingest color names with spaces because they are
9.8+ ;; duplicates - 'dark goldenrod' has the same value as
9.9+ ;; 'darkgoldenrod' so just use that.
9.10 (if (and match (not (find #\space (aref registers 3))))
9.11 (let ((colorname (string-downcase (aref registers 3))))
9.12 (format colordefs
