Mercurial > core / lisp/ffi/rocksdb/opts.lisp
| changeset 679: |
12287fab15d0 |
| parent: |
c7f9bfc9570f
|
| author: |
Richard Westhaver <ellis@rwest.io> |
| date: |
Thu, 26 Sep 2024 21:16:45 -0400 |
| permissions: |
-rw-r--r-- |
| description: |
rocksdb load opts and env updates |
1 ;;; rocksdb/opts.lisp --- Rocksdb Options FFI 8 (define-opt rocksdb-ingestexternalfileoptions) 9 (define-alien-routine rocksdb-ingestexternalfileoptions-set-move-files void 11 (define-alien-routine rocksdb-ingestexternalfileoptions-set-snapshot-consistency void 13 (define-alien-routine rocksdb-ingestexternalfileoptions-set-allow-global-seqno void 15 (define-alien-routine rocksdb-ingestexternalfileoptions-set-allow-blocking-flush void 17 (define-alien-routine rocksdb-ingestexternalfileoptions-set-ingest-behind void 19 (define-alien-routine rocksdb-ingestexternalfileoptions-set-fail-if-not-bottommost-level void 21 (export '(rocksdb-ingestexternalfileoptions-set-move-files 22 rocksdb-ingestexternalfileoptions-set-snapshot-consistency 23 rocksdb-ingestexternalfileoptions-set-allow-global-seqno 24 rocksdb-ingestexternalfileoptions-set-allow-blocking-flush 25 rocksdb-ingestexternalfileoptions-set-ingest-behind 26 rocksdb-ingestexternalfileoptions-set-fail-if-not-bottommost-level)) 28 (define-opt rocksdb-backup-engine-options) 29 (define-alien-routine rocksdb-backup-engine-options-set-backup-dir void 30 (opts (* rocksdb-backup-engine-options)) (backup-dir c-string)) 31 (define-alien-routine rocksdb-backup-engine-options-set-env void 32 (opts (* rocksdb-backup-engine-options)) (val boolean)) 33 (define-opt-accessor rocksdb-backup-engine-options share-table-files) 34 (define-opt-accessor rocksdb-backup-engine-options sync) 35 (define-opt-accessor rocksdb-backup-engine-options destroy-old-data) 36 (define-opt-accessor rocksdb-backup-engine-options backup-log-files) 37 (define-opt-accessor rocksdb-backup-engine-options backup-rate-limit (unsigned 64)) 38 (define-opt-accessor rocksdb-backup-engine-options restore-rate-limit (unsigned 64)) 39 (define-opt-accessor rocksdb-backup-engine-options callback-trigger-interval-size (unsigned 64)) 40 (define-opt-accessor rocksdb-backup-engine-options max-valid-backups-to-open int) 41 (define-opt-accessor rocksdb-backup-engine-options shared-files-with-checksum-naming int) 43 (define-opt rocksdb-restore-options) 44 (define-alien-routine rocksdb-restore-options-set-keep-log-files void 45 (opts (* rocksdb-restore-options)) 48 (define-opt rocksdb-hyper-clock-cache-options) 49 (define-alien-routine rocksdb-hyper-clock-cache-options-set-capacity void 50 (opts (* rocksdb-hyper-clock-cache-options)) 52 (define-alien-routine rocksdb-hyper-clock-cache-options-set-estimated-entry-charge void 53 (opts (* rocksdb-hyper-clock-cache-options)) 55 (define-alien-routine rocksdb-hyper-clock-cache-options-set-num-shard-bits void 56 (opts (* rocksdb-hyper-clock-cache-options)) 58 (define-alien-routine rocksdb-hyper-clock-cache-options-set-memory-allocator void 59 (opts (* rocksdb-hyper-clock-cache-options)) 60 (malloc (* rocksdb-memory-allocator))) 62 (define-opt rocksdb-fifo-compaction-options) 63 (define-opt rocksdb-transactiondb-options) 64 (define-opt rocksdb-transaction-options) 65 (define-opt rocksdb-optimistictransaction-options) 66 (define-opt rocksdb-envoptions) 67 (define-opt rocksdb-universal-compaction-options) 70 (define-opaque rocksdb-wal-readoptions) 72 ;;; Block based Table Options 74 (define-opaque rocksdb-block-based-table-options)) 76 (define-alien-routine rocksdb-block-based-options-create (* rocksdb-block-based-table-options)) 77 (define-alien-routine rocksdb-block-based-options-destroy void (self (* rocksdb-block-based-table-options))) 78 (define-alien-routine rocksdb-block-based-options-set-checksum void 79 (opt (* rocksdb-block-based-table-options)) (val char)) 80 (define-alien-routine rocksdb-block-based-options-set-block-size void 81 (opt (* rocksdb-block-based-table-options)) (block-size size-t)) 82 (define-alien-routine rocksdb-block-based-options-set-block-size-deviation void 83 (opt (* rocksdb-block-based-table-options)) (block-size-deviation int)) 84 (define-alien-routine rocksdb-block-based-options-set-block-restart-interval void 85 (opt (* rocksdb-block-based-table-options)) (block-restart-interval int)) 86 (define-alien-routine rocksdb-block-based-options-set-index-block-restart-interval void 87 (opt (* rocksdb-block-based-table-options)) (index-block-restart-interval char)) 88 (define-alien-routine rocksdb-block-based-options-set-metadata-block-size void 89 (opt (* rocksdb-block-based-table-options)) (metadata-block-size unsigned-long)) 90 (define-alien-routine rocksdb-block-based-options-set-partition-filters void 91 (opt (* rocksdb-block-based-table-options)) (partition-filters unsigned-char)) 92 (define-alien-routine rocksdb-block-based-options-set-partition-filters-for-memory void 93 (opt (* rocksdb-block-based-table-options)) (optimize-filters-for-memory unsigned-char)) 94 (define-alien-routine rocksdb-block-based-options-set-use-delta-encoding void 95 (opt (* rocksdb-block-based-table-options)) (use-delta-encoding unsigned-char)) 96 (define-alien-routine rocksdb-block-based-options-set-no-block-cache void 97 (opt (* rocksdb-block-based-table-options)) (no-block-cache unsigned-char)) 98 (define-alien-routine rocksdb-block-based-options-set-block-cache void 99 (opt (* rocksdb-block-based-table-options)) (block-cache (* rocksdb-cache))) 100 (define-alien-routine rocksdb-block-based-options-set-format-version void 101 (opt (* rocksdb-block-based-table-options)) (val int)) 102 (define-alien-routine rocksdb-block-based-options-set-index-type void 103 (opt (* rocksdb-block-based-table-options)) (val int)) 104 (define-alien-routine rocksdb-block-based-options-set-data-block-index-type void 105 (opt (* rocksdb-block-based-table-options)) (val int)) 106 (define-alien-routine rocksdb-block-based-options-set-data-block-hash-ratio void 107 (opt (* rocksdb-block-based-table-options)) (val double)) 108 (define-alien-routine rocksdb-block-based-options-set-cache-index-and-filter-blocks void 109 (opt (* rocksdb-block-based-table-options)) (val unsigned-char)) 110 (define-alien-routine rocksdb-block-based-options-set-cache-index-and-filter-blocks-with-high-priority void 111 (opt (* rocksdb-block-based-table-options)) (val unsigned-char)) 112 (define-alien-routine rocksdb-block-based-options-set-pin-l0-filter-and-index-blocks-in-cache void 113 (opt (* rocksdb-block-based-table-options)) (val unsigned-char)) 114 (define-alien-routine rocksdb-block-based-options-set-pin-top-level-index-and-filter void 115 (opt (* rocksdb-block-based-table-options)) (val unsigned-char)) 116 (define-alien-routine rocksdb-block-based-options-set-top-level-index-pinning-tier void 117 (opt (* rocksdb-block-based-table-options)) 119 (define-alien-routine rocksdb-block-based-options-set-partition-pinning-tier void 120 (opt (* rocksdb-block-based-table-options)) 122 (define-alien-routine rocksdb-block-based-options-set-unpartition-pinning-tier void 123 (opt (* rocksdb-block-based-table-options)) 126 (export-opt-accessors rocksdb-block-based-options 130 block-restart-interval 131 index-block-restart-interval 134 partition-filters-for-memory 140 data-block-index-type 141 data-block-hash-ratio 142 cache-index-and-filter-blocks 143 cache-index-and-filter-blocks-with-high-priority 144 pin-l0-filter-and-index-blocks-in-cache 145 pin-top-level-index-and-filter) 147 ;;; Cuckoo Table Options 148 (define-opaque rocksdb-cuckoo-table-options) 151 (define-opt rocksdb-options) 152 (define-opt-accessor rocksdb-options create-if-missing) 153 (define-opt-accessor rocksdb-options create-missing-column-families) 154 (define-opt-accessor rocksdb-options error-if-exists) 155 (define-opt-accessor rocksdb-options paranoid-checks) 156 (define-opt-accessor rocksdb-options compression-options-use-zstd-dict-trainer) 157 (define-opt-accessor rocksdb-options level-compaction-dynamic-level-bytes (unsigned 8)) 158 (define-opt-accessor rocksdb-options enable-blob-gc) 159 (define-opt-accessor rocksdb-options allow-ingest-behind) 160 (define-opt-accessor rocksdb-options skip-stats-update-on-db-open) 161 (define-opt-accessor rocksdb-options skip-checking-sst-file-sizes-on-db-open) 162 (define-opt-accessor rocksdb-options enable-blob-files) 163 (define-opt-accessor rocksdb-options enable-pipelined-write) 164 (define-opt-accessor rocksdb-options unordered-write) 165 (define-opt-accessor rocksdb-options allow-mmap-reads) 166 (define-opt-accessor rocksdb-options allow-mmap-writes) 167 (define-opt-accessor rocksdb-options use-direct-reads) 168 (define-opt-accessor rocksdb-options use-direct-io-for-flush-and-compaction) 169 (define-opt-accessor rocksdb-options is-fd-close-on-exec) 170 (define-opt-accessor rocksdb-options inplace-update-num-locks size-t) 171 (define-opt-accessor rocksdb-options inplace-update-support) 172 (define-opt-accessor rocksdb-options advise-random-on-open) 173 (define-opt-accessor rocksdb-options atomic-flush) 174 (define-opt-accessor rocksdb-options manual-wal-flush) 175 (define-opt-accessor rocksdb-options avoid-unnecessary-blocking-io) 176 (define-opt-accessor rocksdb-options writable-file-max-buffer-size (unsigned 64)) 177 (define-opt-accessor rocksdb-options info-log-level int) 178 (define-opt-accessor rocksdb-options write-buffer-size size-t) 179 (define-opt-accessor rocksdb-options db-write-buffer-size size-t) 180 (define-opt-accessor rocksdb-options max-open-files int) 181 (define-opt-accessor rocksdb-options max-file-opening-threads int) 182 (define-opt-accessor rocksdb-options max-total-wal-size unsigned-long) 183 (define-opt-accessor rocksdb-options compression-options-zstd-max-train-bytes int) 184 (define-opt-accessor rocksdb-options compression-options-max-dict-buffer-bytes unsigned-long) 185 (define-opt-accessor rocksdb-options num-levels int) 186 (define-opt-accessor rocksdb-options level0-file-num-compaction-trigger int) 187 (define-opt-accessor rocksdb-options level0-slowdown-writes-trigger int) 188 (define-opt-accessor rocksdb-options level0-stop-writes-trigger int) 189 (define-opt-accessor rocksdb-options target-file-size-base unsigned-long) 190 (define-opt-accessor rocksdb-options target-file-size-multiplier int) 191 (define-opt-accessor rocksdb-options max-bytes-for-level-base unsigned-long) 192 (define-opt-accessor rocksdb-options max-bytes-for-level-multiplier double) 194 (define-alien-routine rocksdb-options-set-compression-options void 195 (opt (* rocksdb-options)) 196 (a int) (b int) (c int) (d int)) 198 (define-alien-routine rocksdb-option-set-block-based-table-factory void 199 (opt (* rocksdb-options)) (table-opts (* rocksdb-block-based-table-options))) 201 (define-opt-accessor rocksdb-options comparator (* rocksdb-comparator)) 202 (define-opt-accessor rocksdb-options merge-operator (* rocksdb-mergeoperator)) 203 (define-opt-accessor rocksdb-options statistics-level int) 204 (define-opt-accessor rocksdb-options min-blob-size unsigned-long) 205 (define-opt-accessor rocksdb-options blob-file-size unsigned-long) 206 (define-opt-accessor rocksdb-options blob-compression-type int) 208 (define-opt-accessor rocksdb-options blob-gc-age-cutoff double) 209 (define-opt-accessor rocksdb-options blob-gc-force-threshold double) 210 (define-opt-accessor rocksdb-options blob-compaction-readahead-size unsigned-long) 211 (define-opt-accessor rocksdb-options blob-file-starting-level int) 212 (define-opt-accessor rocksdb-options blob-cache (* rocksdb-cache)) 213 (define-opt-accessor rocksdb-options prepopulate-blob-cache int) 214 (define-opt-accessor rocksdb-options max-write-buffer-number int) 215 (define-opt-accessor rocksdb-options min-write-buffer-number-to-merge int) 216 (define-opt-accessor rocksdb-options max-write-buffer-number-to-maintain int) 217 (define-opt-accessor rocksdb-options max-write-buffer-size-to-maintain long) 218 (define-opt-accessor rocksdb-options max-subcompactions unsigned-int) 219 (define-opt-accessor rocksdb-options max-background-jobs int) 220 (define-opt-accessor rocksdb-options max-background-compactions int) 221 (define-opt-accessor rocksdb-options max-background-flushes int) 222 (define-opt-accessor rocksdb-options max-log-file-size size-t) 223 (define-opt-accessor rocksdb-options log-file-time-to-roll size-t) 224 (define-opt-accessor rocksdb-options keep-log-file-num size-t) 225 (define-opt-accessor rocksdb-options recycle-log-file-num size-t) 226 (define-opt-accessor rocksdb-options soft-pending-compaction-bytes-limit size-t) 227 (define-opt-accessor rocksdb-options hard-pending-compaction-bytes-limit size-t) 228 (define-opt-accessor rocksdb-options max-manifest-file-size size-t) 229 (define-opt-accessor rocksdb-options table-cache-numshardbits int) 230 (define-opt-accessor rocksdb-options arena-block-size size-t) 232 ;; may need to be an int -- check src 233 (define-opt-accessor rocksdb-options use-fsync boolean) 234 (define-alien-routine rocksdb-options-set-db-log-dir void 235 (opts (* rocksdb-options)) 237 (define-alien-routine rocksdb-options-set-wal-dir void 238 (opts (* rocksdb-options)) 240 (define-opt-accessor rocksdb-options wal-ttl-seconds unsigned-long) 241 (define-opt-accessor rocksdb-options wal-size-limit-mb unsigned-long) 242 (define-opt-accessor rocksdb-options manifest-preallocation-size size-t) 243 (define-opt-accessor rocksdb-options stats-dump-period-sec unsigned-int) 244 (define-opt-accessor rocksdb-options stats-persist-period-sec unsigned-int) 246 (define-opt-accessor rocksdb-options use-adaptive-mutex) 247 (define-opt-accessor rocksdb-options bytes-per-sync unsigned-long) 248 (define-opt-accessor rocksdb-options wal-bytes-per-sync unsigned-long) 249 (define-opt-accessor rocksdb-options file-max-buffer-size unsigned-long) 250 (define-opt-accessor rocksdb-options allow-concurrent-memtable-write) 251 (define-opt-accessor rocksdb-options enable-write-thread-adaptive-yield) 252 (define-opt-accessor rocksdb-options max-sequential-skip-in-iterations unsigned-long) 253 (define-opt-accessor rocksdb-options disable-auto-compactions) 254 (define-opt-accessor rocksdb-options optimize-filters-for-hits) 255 (define-opt-accessor rocksdb-options delete-obsolete-files-period-micros unsigned-long) 256 (define-opt-accessor rocksdb-options memtable-prefix-bloom-size-ratio double) 257 (define-opt-accessor rocksdb-options max-compaction-bytes unsigned-long) 258 (define-opt-accessor rocksdb-options memtable-huge-page-size size-t) 259 (define-opt-accessor rocksdb-options max-successive-merges size-t) 260 (define-opt-accessor rocksdb-options bloom-locality unsigned-int) 261 (define-opt-accessor rocksdb-options report-bg-io-stats) 262 (define-opt-accessor rocksdb-options experimental-mempurge-threshold double) 263 (define-opt-accessor rocksdb-options wal-recovery-mode int) 264 (define-opt-accessor rocksdb-options compression-options-parallel-threads int) 265 (define-opt-accessor rocksdb-options compression int) 266 (define-opt-accessor rocksdb-options bottommost-compression int) 267 (define-opt-accessor rocksdb-options compaction-style int) 268 (define-opt-accessor rocksdb-options wal-compression int) 270 rocksdb_k_by_compensated_size_compaction_pri = 0, 271 rocksdb_k_oldest_largest_seq_first_compaction_pri = 1, 272 rocksdb_k_oldest_smallest_seq_first_compaction_pri = 2, 273 rocksdb_k_min_overlapping_ratio_compaction_pri = 3, 274 rocksdb_k_round_robin_compaction_pri = 4 276 (define-opt-accessor rocksdb-options compaction-pri int) 277 ;; (hash-link-list-rep) 278 ;; (hash-skip-list-rep) 279 ;; (memtable-vector-rep) 281 (define-alien-routine rocksdb-options-set-row-cache void 282 (opt (* rocksdb-options)) 283 (cache (* rocksdb-cache))) 285 (define-alien-routine rocksdb-options-set-ratelimiter void 286 (opt (* rocksdb-options)) 287 (limiter (* rocksdb-ratelimiter))) 289 (define-alien-routine rocksdb-options-set-universal-compaction-options void 290 (opt (* rocksdb-options)) 291 (opts (* rocksdb-universal-compaction-options))) 293 (define-alien-routine rocksdb-options-set-min-level-to-compress void 294 (opt (* rocksdb-options)) 297 (define-alien-routine rocksdb-options-set-plain-table-factory void 298 (opt (* rocksdb-options)) 307 (define-alien-routine rocksdb-options-prepare-for-bulk-load void 308 (opts (* rocksdb-options))) 310 (define-alien-routine rocksdb-options-increase-parallelism void 311 (opt (* rocksdb-options)) (total-threads int)) 313 (define-alien-routine rocksdb-options-optimize-level-style-compaction void 314 (opt (* rocksdb-options)) 315 (memtable-memory-budget unsigned-long)) 317 (define-alien-routine rocksdb-options-enable-statistics void 318 (opt (* rocksdb-options))) 320 (define-alien-routine rocksdb-options-statistics-get-string c-string 321 (opt (* rocksdb-options))) 323 (define-alien-routine rocksdb-options-statistics-get-ticker-count unsigned-long 324 (opt (* rocksdb-options)) 325 (ticker-type unsigned-int)) 327 (define-alien-routine rocksdb-options-statistics-get-histogram-data void 328 (opt (* rocksdb-options)) 329 (histogram-type unsigned-int) 330 (data (* rocksdb-statistics-histogram-data))) 332 (define-alien-routine rocksdb-options-set-db-paths void 333 (opt (* rocksdb-options)) 334 (paths (array (* rocksdb-dbpath))) 337 (define-alien-routine rocksdb-options-set-cf-paths void 338 (opt (* rocksdb-options)) 339 (paths (array (* rocksdb-dbpath))) 342 (define-alien-routine rocksdb-options-set-env void 343 (opts (* rocksdb-options)) 344 (env (* rocksdb-env))) 346 (define-alien-routine rocksdb-options-set-info-log void 347 (opts (* rocksdb-options)) 348 (logger (* rocksdb-logger))) 350 (define-alien-routine rocksdb-options-set-uint64add-merge-operator void 351 (opt (* rocksdb-options))) 353 (define-alien-routine rocksdb-options-set-compression-per-level void 354 (opt (* rocksdb-options)) 358 ;;; RocksDB Write Options 359 (define-opt rocksdb-writeoptions) 360 ;;; RocksDB Read Options 361 (define-opt rocksdb-readoptions) 362 ;;; RocksDB Flush Options 363 (define-opt rocksdb-flushoptions) 364 (define-opt-accessor rocksdb-flushoptions wait) 365 ;;; RocksDB Compact Options 366 (define-opt rocksdb-compactoptions) 367 (define-opt-accessor rocksdb-compactoptions exclusive-manual-compaction) 368 (define-opt-accessor rocksdb-compactoptions bottommost-level-compaction) 369 (define-opt-accessor rocksdb-compactoptions change-level) 370 (define-opt-accessor rocksdb-compactoptions target-level int) 371 ;;; RocksDB LRU Cache Options 372 (define-opt rocksdb-lru-cache-options) 374 (define-alien-routine rocksdb-lru-cache-options-set-capacity void 375 (self (* rocksdb-lru-cache-options)) 378 (define-alien-routine rocksdb-lru-cache-options-set-num-shard-bits void 379 (self (* rocksdb-lru-cache-options)) 382 (define-alien-routine rocksdb-lru-cache-options-set-memory-allocator void 383 (self (* rocksdb-lru-cache-options)) 384 (val (* rocksdb-memory-allocator))) 386 (export-opt-accessors rocksdb-lru-cache-options 392 Load the latest rocksdb options from the specified db_path. 394 On success, num_column_families will be updated with a non-zero 395 number indicating the number of column families. 396 The returned db_options, column_family_names, and column_family_options 397 should be released via rocksdb_load_latest_options_destroy(). 399 On error, a non-null errptr that includes the error message will be 400 returned. db_options, column_family_names, and column_family_options 403 (def-with-errptr rocksdb-load-latest-options 406 (env (* rocksdb-env)) 407 (ignore-unknown-options boolean) 408 (cache (* rocksdb-cache)) 409 (db-options (* (* rocksdb-options))) 410 (num-column-families (* size-t)) 411 (column-family-names (* (* c-string))) 412 (column-family-options (* (* (* rocksdb-options))))) 414 (define-alien-routine rocksdb-load-latest-options-destroy void 415 (db-options (* rocksdb-options)) 416 (list-column-family-names (* c-string)) 417 (list-column-family-options (* (* rocksdb-options))) 420 (def-with-errptr rocksdb-set-options void 423 (keys (array c-string)) 424 (values (array c-string))) 426 (def-with-errptr rocksdb-set-options-cf 429 (handle (* rocksdb-column-family-handle)) 431 (keys (array (array unsigned-char))) 432 (values (array (array unsigned-char)))) 434 (define-alien-routine rocksdb-options-create-copy (* rocksdb-options) 435 (src (* rocksdb-options))) 438 ;; some of the RocksDB options don't follow the standard naming 439 ;; convention of 'rocksdb-*-set-*' and 'rocksdb-*-get-*'. In order to 440 ;; remove the need for special-case handling in the high-level 441 ;; interface we define them as aliases 442 (setf (symbol-function 'rocksdb-options-set-parallelism) #'rocksdb-options-increase-parallelism) 444 (declaim (inline rocksdb-options-set-enable-statistics rocksdb-options-set-prepare-for-bulk-load)) 445 (defun rocksdb-options-set-enable-statistics (opt x) 447 (rocksdb-options-enable-statistics opt))) 449 (defun rocksdb-options-set-prepare-for-bulk-load (opt x) 451 (rocksdb-options-prepare-for-bulk-load opt)))
