changeset 658: | 804b5ee20a46 |
parent: | c6caddf91c72 |
author: | Richard Westhaver <ellis@rwest.io> |
date: | Thu, 19 Sep 2024 23:23:02 -0400 |
permissions: | -rw-r--r-- |
description: | zstd completed (besides zdict), working on readline |
400 | 1 | ;;; io/flate.lisp --- Compressed IO Interface |
2 | ||
3 | ;; Use compression (ZSTD) with Lisp objects and streams. |
|
4 | ||
5 | ;;; Commentary: |
|
6 | ||
7 | ;; compression ref: https://www.xach.com/lisp/salza2/ (compression only) |
|
8 | ||
9 | ;; decompression ref: https://github.com/sharplispers/chipz (decompression only) |
|
10 | ||
11 | ;; The libraries above are the current state-of-the-art for compression and |
|
12 | ;; decompression in Common Lisp. They are portable packages which depend on |
|
13 | ;; gray streams. They loosely cover deflate, zlib, gzip, and bzip2 data. |
|
14 | ||
15 | ;; The compression backends are themselves hand-coded in Common Lisp, making |
|
16 | ;; them excellent reference material. However, we don't have much use for the |
|
658
804b5ee20a46
zstd completed (besides zdict), working on readline
Richard Westhaver <ellis@rwest.io>
parents:
470
diff
changeset
|
17 | ;; compression backends offered. |
400 | 18 | |
19 | ;; We intend to almost exclusively support Zstd compression and decompression |
|
20 | ;; using our ZSTD FFI Lisp system, so we'll make a new library - FLATE - which |
|
21 | ;; provides a shared zstd compression/decompression to Lisp objects and |
|
22 | ;; streams. |
|
23 | ||
24 | ;;; Code: |
|
25 | (in-package :io/flate) |
|
26 | ||
27 | ;;; Vars |
|
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
28 | (defparameter *compression-buffer-size* 4096) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
29 | (defparameter *decompression-buffer-size* 4096) |
470 | 30 | (defparameter *default-compression-level* (zstd:zstd-defaultclevel)) |
31 | ||
32 | (defvar *compression-level*) |
|
400 | 33 | |
34 | ;;; Utils |
|
35 | ||
470 | 36 | ;;; Errors |
435
849bbe48e32d
added dat/mime, removed sans-io
Richard Westhaver <ellis@rwest.io>
parents:
431
diff
changeset
|
37 | (eval-always (deferror flate-error () () (:auto t))) |
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
38 | |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
39 | (deferror compression-error (flate-error) () (:auto t)) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
40 | (deferror decompression-error (flate-error) () (:auto t)) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
41 | |
470 | 42 | ;;; Proto |
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
43 | (defgeneric finish-compression (self)) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
44 | (defgeneric finish-decompression (self)) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
45 | ;; TODO 2024-06-08: maybe move this to generic io/stream protocol - 'RESET' |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
46 | (defgeneric reset-compressor (self)) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
47 | (defgeneric reset-decompressor (self)) |
441
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
48 | (defgeneric make-compressed-stream (&optional stream)) |
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
49 | (defgeneric make-decompressed-stream (&optional stream)) |
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
50 | (defgeneric compress-object (self)) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
51 | (defgeneric decompress-object (self)) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
52 | |
470 | 53 | (defgeneric compress (input output state)) |
54 | (defgeneric decompress (input output state)) |
|
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
55 | |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
56 | ;; decompress |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
57 | |
400 | 58 | ;;; Compression |
59 | ||
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
60 | ;; AKA 'DEFLATE' |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
61 | |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
62 | ;; compress-octet |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
63 | ;; compress-octet-vector |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
64 | |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
65 | ;; finish-compression (finish-output?) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
66 | ;; with-compressor |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
67 | ;; reset-compressor |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
68 | |
441
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
69 | ;; make-compressed-stream |
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
70 | |
441
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
71 | (defclass compressor () ()) |
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
72 | |
441
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
73 | (defclass compressed-stream (fundamental-binary-stream) |
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
74 | ()) |
400 | 75 | |
76 | ;;; Decompression |
|
77 | ||
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
78 | ;; AKA 'INFLATE' |
400 | 79 | |
80 | ;; From chipz: |
|
81 | ;; We provide several convenience functions for decompression: |
|
82 | ;; |
|
83 | ;; * decompress a buffer to a newly-consed buffer; |
|
84 | ;; * decompress a stream to a newly-consed buffer; |
|
85 | ;; * decompress a pathname to a newly-consed buffer; |
|
86 | ;; * decompress a buffer to a user-specified buffer; |
|
87 | ;; * decompress a buffer to a stream; |
|
88 | ;; * decompress a stream to a stream. |
|
89 | ;; * decompress a pathname to another pathname; |
|
90 | ;; * decompress a pathname to a stream; |
|
91 | ;; |
|
92 | ;; We do not provide stream->buffer decompression, as we have no way of |
|
93 | ;; knowing how much to read from the stream to fill the buffer, no way |
|
94 | ;; of determining what to do with possible state left in the |
|
95 | ;; INFLATE-STATE that we used, etc. Application-specific logic will |
|
96 | ;; have to handle those bits. |
|
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
97 | |
441
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
98 | ;; make-decompressed-stream |
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
99 | ;; decompress-octet |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
100 | ;; decompress-octet-vector |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
101 | |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
102 | (defclass decompressor () ()) |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
103 | |
441
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
104 | (defclass decompressed-stream (fundamental-binary-stream) |
9fa3b9154bb2
add pod/containerfile, rm pod/buildah, some work on zstd
Richard Westhaver <ellis@rwest.io>
parents:
435
diff
changeset
|
105 | ()) |
431
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
106 | |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
107 | ;;; API |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
108 | |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
109 | ;; zstd-stream |
c40d2a41d7ce
source concatenating std.lisp, more systems, got zstd simple working, IO work, added dat/tar
Richard Westhaver <ellis@rwest.io>
parents:
400
diff
changeset
|
110 | ;; zstd-file |