changelog shortlog graph tags branches changeset files revisions annotate raw help

Mercurial > core / lisp/ffi/zstd/pkg.lisp

changeset 657: 937a6f354047
parent: 4dd7b6320efc
child: 08621be7e780
author: Richard Westhaver <ellis@rwest.io>
date: Wed, 18 Sep 2024 21:48:06 -0400
permissions: -rw-r--r--
description: zstd tests and macros 
     1 ;;; ffi/zstd/pkg.lisp --- ZSTD FFI
     2 
     3 ;; Zstd compression support for Lisp
     4 
     5 ;;; Commentary:
     6 
     7 ;; Initially I was thinking of this as an SB-CONTRIB module which links up
     8 ;; with whatever C runtime functions exposed by the built-in SBCL compression
     9 ;; support. However, there isn't actually much going on in the runtime and
    10 ;; it's not publicly exposed at all. The SBCL/Zstd surface-area is restrained
    11 ;; to FASL read/write streams and not of much use outside it.
    12 
    13 ;; So, we'll be applying the same from-scratch strategy we've become
    14 ;; accustomed to, exposing as much of the C API as possible and building up
    15 ;; our abstractions.
    16 
    17 ;; The low-level abstractions are in this package - ZSTD.
    18 
    19 ;; For the high-level abstractions see IO/FLATE and IO/ZSTD in the IO package.
    20 
    21 ;; The following programs have compile-time support for linking Zstd:
    22 
    23 ;; SBCL
    24 ;; QEMU
    25 ;; RocksDB 
    26 
    27 #| from zstd.h:
    28 Introduction                            ; ; ; ; ; ;
    29                                         ; ; ; ; ; ;
    30 zstd, short for Zstandard, is a fast lossless compression algorithm, targeting ; ; ; ; ; ;
    31 real-time compression scenarios at zlib-level and better compression ratios. ; ; ; ; ; ;
    32 The zstd compression library provides in-memory compression and decompression ; ; ; ; ; ;
    33 functions.                              ; ; ; ; ; ;
    34                                         ; ; ; ; ; ;
    35 The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), ; ; ; ; ; ;
    36 which is currently 22. Levels >= 20, labeled `--ultra`, should be used with ; ; ; ; ; ;
    37 caution, as they require more memory. The library also offers negative ; ; ; ; ; ;
    38 compression levels, which extend the range of speed vs. ratio preferences. ; ; ; ; ; ;
    39 The lower the level, the faster the speed (at the cost of compression). ; ; ; ; ; ;
    40                                         ; ; ; ; ; ;
    41 Compression can be done in:             ; ; ; ; ; ;
    42 - a single step (described as Simple API) ; ; ; ; ; ;
    43 - a single step, reusing a context (described as Explicit context) ; ; ; ; ; ;
    44 - unbounded multiple steps (described as Streaming compression) ; ; ; ; ; ;
    45                                         ; ; ; ; ; ;
    46 The compression ratio achievable on small data can be highly improved using ; ; ; ; ; ;
    47 a dictionary. Dictionary compression can be performed in: ; ; ; ; ; ;
    48 - a single step (described as Simple dictionary API) ; ; ; ; ; ;
    49 - a single step, reusing a dictionary (described as Bulk-processing ; ; ; ; ; ;
    50 dictionary API)                         ; ; ; ; ; ;
    51                                         ; ; ; ; ; ;
    52 Advanced experimental functions can be accessed using ; ; ; ; ; ;
    53 `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. ; ; ; ; ; ;
    54                                         ; ; ; ; ; ;
    55 Advanced experimental APIs should never be used with a dynamically-linked ; ; ; ; ; ;
    56 library. They are not "stable"; their definitions or signatures may change in ; ; ; ; ; ;
    57 the future. Only static linking is allowed. ; ; ; ; ; ;
    58 |#
    59 
    60 ;;; Code:
    61 (defpackage :zstd
    62   (:use :cl :std :sb-alien)
    63   (:nicknames :zstd)
    64   (:export :zstd-alien-error :with-zstd-cstream :with-zstd-dstream
    65    :zstd-versionnumber :zstd-cstreaminsize :zstd-cstreamoutsize :zstd-inbuffer
    66    :zstd-iserror :zstd-defaultclevel :zstd-compress :zstd-decompress
    67    :zstd-cstream :zstd-dstream :zstd-compressstream :zstd-decompressstream
    68    :zstd-compressstream2 :zstd-outbuffer :zstd-geterrorname :zstd-geterrorcode
    69    :zstdc :zstdd
    70    :zstd-alien-error :zstd-dstream-error :zstd-cstream-error
    71    :with-zstd-streams
    72    :with-zstd-buffers
    73    :with-zstd-outbuffer
    74    :with-zstd-inbuffer
    75    :with-zstd-cdict
    76    :with-zstd-ddict))
    77 
    78 (in-package :zstd)
    79 
    80 (define-alien-loader "zstd" t "/usr/lib/")
    81 
    82 ;;; Types
    83 (deftype zstd-error-code ()
    84   `(integer 0 120))
    85 
    86 (deftype zstd-strategy-designator ()
    87   `(or (integer ,(zstd-minclevel) ,(zstd-maxclevel))
    88        (member :fast :dfast :greedy :lazy
    89                :lazy2 :btlazy2 :btopt :btultra
    90                :btultra2)))
    91 
    92 (deftype zstd-compression-parameter ()
    93   `(integer 100 1024))
    94 (deftype zstd-decompression-parameter ()
    95   `(integer 100 1024))
    96 
    97 (deftype zstd-reset-directive ()
    98   `(or (integer 1 3) (member :session-only :parameters :session-and-parameters)))
    99 (deftype zstd-end-directive ()
   100   `(or (integer 0 2) (member :continue :flus :end)))
   101 
   102 ;;; Errors
   103 (define-condition zstd-alien-condition () ()
   104   (:documentation "Superclass of all conditions triggered by the ZSTD FFI."))
   105 
   106 (deferror zstd-alien-error (error)
   107     ((code :initarg :code :accessor zstd-error-code))
   108     (:documentation "Error signaled from Zstd Alien."))
   109     
   110 ;; found in zstd_errors.h
   111 (define-alien-enum (zstd-errorcode int)
   112                    :no-error 0
   113                    :generic 1
   114                    :prefix-unknown 10
   115                    :version-unsupported 12
   116                    :frameparameter-unsupported 14
   117                    :frameparameter-windowtoolarge 16
   118                    :corruption-detected 20
   119                    :checksum-wrong 22
   120                    :literals-headerwrong 24
   121                    :dictionary-corrupted 30
   122                    :dictionary-wrong 32
   123                    :dictionarycreation-failed 34
   124                    :parameter-unsupported 40
   125                    :parameter-combination-unsupported 41
   126                    :parameter-outofbound 42
   127                    :tablelog-toolarge 44
   128                    :maxsymbolvalue-toolarge 46
   129                    :maxsymbolvalue-toosmall 48
   130                    :stabilitycondition-notrespected 50
   131                    :stage-wrong 60
   132                    :init-missing 62
   133                    :memory-allocation 64
   134                    :workspace-toosmall 66
   135                    :dstsize-toosmall 70
   136                    :srcsize-wrong 72
   137                    :dstbuffer-null 74
   138                    :noforwardprogress-destfull 80
   139                    :noforwardprogress-inputempty 82
   140                    ;; unstable
   141                    :frameindex-toolarge 100
   142                    :seekableio 102
   143                    :dstbuffer-wrong 104
   144                    :srcbuffer-wrong 105
   145                    :sequenceproducer-failed 106
   146                    :externalsequences-invalid 107
   147                    :maxcode 120)
   148 
   149 ;;; Utils
   150 (define-alien-routine "ZSTD_versionNumber" unsigned)
   151 (define-alien-routine "ZSTD_versionString" c-string)
   152 (define-alien-routine "ZSTD_compressBound" size-t (src-size size-t))
   153 (define-alien-routine "ZSTD_isError" unsigned (code size-t))
   154 (define-alien-routine "ZSTD_getErrorName" c-string (code size-t))
   155 ;; zstd_errors.h - does this work?
   156 (define-alien-routine "ZSTD_getErrorCode" int (function-result size-t))
   157 (define-alien-routine "ZSTD_getErrorString" c-string (code int))
   158 
   159 (define-alien-routine "ZSTD_minCLevel" int)
   160 (define-alien-routine "ZSTD_maxCLevel" int)
   161 (define-alien-routine "ZSTD_defaultCLevel" int)
   162 
   163 (define-alien-routine "ZSTD_findFrameCompressedSize" size-t
   164   (src (* t))
   165   (src-size size-t))
   166 
   167 (define-alien-routine "ZSTD_getFrameContentSize" unsigned-long-long
   168   (src (* t))
   169   (src-size size-t))
   170 
   171 (define-alien-routine "ZSTD_decompressBound" unsigned-long-long
   172   (src (* t))
   173   (src-size size-t))
   174 
   175 ;;; Explicit Context API
   176 (define-alien-type zstd-cctx (struct zstd-cctx-s))
   177 
   178 (define-alien-routine "ZSTD_createCCtx" (* zstd-cctx))
   179 (define-alien-routine "ZSTD_freeCCtx" void (cctx (* zstd-cctx)))
   180 (define-alien-routine "ZSTD_compressCCtx" size-t
   181   (cctx (* zstd-cctx))
   182   (dst (* t)) (dst-capacity size-t)
   183   (src (* t)) (src-size size-t)
   184   (compression-level int))
   185 
   186 (define-alien-type zstd-dctx (struct zstd-dctx-s))
   187 
   188 (define-alien-routine "ZSTD_createDCtx" (* zstd-dctx))
   189 (define-alien-routine "ZSTD_freeDCtx" void (dctx (* zstd-dctx)))
   190 (define-alien-routine "ZSTD_decompressDCtx" size-t
   191   (dctx (* zstd-dctx))
   192   (dst (* t)) (dst-capacity size-t)
   193   (src (* t)) (src-size size-t))
   194 ;;; Advanced API
   195 (define-alien-enum (zstd-strategy int)
   196                    :fast 1
   197                    :dfast 2
   198                    :greedy 3
   199                    :lazy 4
   200                    :lazy2 5
   201                    :btlazy2 6
   202                    :btopt 7
   203                    :btultra 8
   204                    :btultra2 9)
   205 
   206 (define-alien-enum (zstd-cparameter int)
   207                    :compression-level 100
   208                    :window-log 101
   209                    :hash-log 102
   210                    :chain-log 103
   211                    :search-log 104
   212                    :min-match 105
   213                    :target-length 106
   214                    :strategy 107
   215                    :target-c-block-size 130
   216                    :enable-long-distance-matching 160
   217                    :ldm-hash-log 161
   218                    :ldm-min-match 162
   219                    :ldm-bucket-size-log 163
   220                    :ldm-hash-rate-log 164
   221                    :content-size-flag 200
   222                    :checksum-flag 201
   223                    :dict-id-flag 202
   224                    :nb-workers 400
   225                    :job-size 401
   226                    :overlap-log 402
   227                    :expiremental1 500
   228                    :expiremental2 10
   229                    :expiremental3 1000
   230                    :expiremental4 1001
   231                    :expiremental5 1002
   232                    ;; :expiremental6 1003 ;; is now target-c-block-size
   233                    :expiremental7 1004
   234                    :expiremental8 1005
   235                    :expiremental9 1006
   236                    :expiremental10 1007
   237                    :expiremental11 1008
   238                    :expiremental12 1009
   239                    :expiremental13 1010
   240                    :expiremental14 1011
   241                    :expiremental15 1012
   242                    :expiremental16 1013
   243                    :expiremental17 1014
   244                    :expiremental18 1015
   245                    :expiremental19 1016)
   246 
   247 (define-alien-enum (zstd-reset-directive int)
   248                    :session-only 1
   249                    :parameters 2
   250                    :session-and-parameters 3)
   251 
   252 (define-alien-enum (zstd-dparameter int)
   253                    :window-log-max 100
   254                    :experimental1 1000
   255                    :experimental2 1001
   256                    :experimental3 1002
   257                    :experimental4 1003                   
   258                    :experimental5 1004
   259                    :experimental6 1005)