Mercurial > core / lisp/lib/io/fast.lisp
changeset 695: |
2bad47888dbf |
parent: |
90417ae14b21
|
author: |
Richard Westhaver <ellis@rwest.io> |
date: |
Fri, 04 Oct 2024 16:14:44 -0400 |
permissions: |
-rw-r--r-- |
description: |
add static-vector |
1 ;;; fast.lisp --- Fast Octet Streams 3 ;; based on https://github.com/rpav/fast-io/tree/master 10 (defvar *default-output-buffer-size* 16) 12 (declaim (ftype (function (array-index) octet-vector) make-octet-vector) 13 (inline make-octet-vector)) 14 (defun make-octet-vector (len) 15 (make-array (the array-index len) :element-type 'octet)) 17 (declaim (inline output-buffer-vector output-buffer-fill output-buffer-len)) 18 (defstruct output-buffer 19 (vector (make-octet-vector *default-output-buffer-size*) 21 (fill 0 :type array-index) 22 (len 0 :type array-index) 23 (queue nil :type list) 27 (defstruct input-buffer 28 (vector nil :type (or null octet-vector)) 29 (pos 0 :type array-index) 32 (defun buffer-position (buffer) 33 "Return the number of bytes read (for an INPUT-BUFFER) or written 34 (for an OUTPUT-BUFFER)" 36 (input-buffer (input-buffer-pos buffer)) 37 (output-buffer (output-buffer-len buffer)))) 39 ;; Sometimes it is usefull just to skip the buffer instead of reading from it. 40 (defun (setf buffer-position) (new-pos buffer) 41 "Set the buffer position for input-buffer" 42 (check-type buffer input-buffer) 43 (let* ((pos (input-buffer-pos buffer)) 44 (vec (input-buffer-vector buffer)) 45 (vec-len (length vec))) 46 (declare (optimize (speed 3) (safety 1)) 47 (type octet-vector vec) 48 (type non-negative-fixnum pos vec-len new-pos)) 49 ;; Only need to update if pos or new-pos is in stream range. 50 (when-let ((stream (and 53 (input-buffer-stream buffer)))) 54 (let* ((stream-file-pos (file-position stream)) 55 (pos-diff (- new-pos pos)) 56 (stream-diff (cond ((and (> pos vec-len) 58 ;; branch for pos in stream and new-pos 63 ;; branch for pos in vector. and new-pos 65 (- pos-diff (- vec-len pos))) 66 ;; otherwise stream-diff = pos-diff. 68 (new-stream-pos (+ stream-file-pos stream-diff))) 69 (declare (type non-negative-fixnum stream-file-pos new-stream-pos) 70 (type fixnum pos-diff stream-diff)) 71 (file-position stream new-stream-pos)))) 72 (setf (slot-value buffer 'pos) new-pos)) 74 (defun octets-from (sequence) 75 (let ((vec (make-octet-vector (length sequence)))) 76 (replace vec sequence) 79 (defun concat-buffer (buffer) 80 (let* ((len (output-buffer-len buffer)) 82 (if (eq :static (output-buffer-output buffer)) 83 (make-static-vector (the array-index len)) 84 (make-octet-vector len)))) 85 (loop as i = 0 then (+ i (length a)) 86 for a in (output-buffer-queue buffer) do 87 (replace (the octet-vector array) 88 (the octet-vector a) :start1 i) 90 (replace (the octet-vector array) 91 (output-buffer-vector buffer) 93 :end2 (output-buffer-fill buffer))) 96 (defun flush (output-buffer) 97 (when (> (output-buffer-fill output-buffer) 0) 98 (write-sequence (output-buffer-vector output-buffer) 99 (output-buffer-output output-buffer) 100 :start 0 :end (output-buffer-fill output-buffer)) 101 (prog1 (output-buffer-fill output-buffer) 102 (setf (output-buffer-fill output-buffer) 0)))) 104 (defun extend (buffer &optional (min 1)) 105 (let ((vector (output-buffer-vector buffer))) 106 (setf (output-buffer-last buffer) 107 (nconc (output-buffer-last buffer) 109 (output-buffer-vector buffer) 110 (make-octet-vector (max min (1+ (* 2 (length vector))))) 111 (output-buffer-fill buffer) 0) 112 (unless (output-buffer-queue buffer) 113 (setf (output-buffer-queue buffer) 114 (output-buffer-last buffer))))) 116 (defun fast-write-byte (byte output-buffer) 117 (declare (type octet byte) 118 (type output-buffer output-buffer) 119 (optimize (speed 3) (safety 1))) 120 (when (= (output-buffer-fill output-buffer) 121 (array-dimension (output-buffer-vector output-buffer) 0)) 122 (if (streamp (output-buffer-output output-buffer)) 123 (flush output-buffer) 124 (extend output-buffer))) 126 (setf (aref (output-buffer-vector output-buffer) 127 (output-buffer-fill output-buffer)) 129 (incf (output-buffer-fill output-buffer)) 130 (incf (output-buffer-len output-buffer)))) 132 (defun fast-read-byte (input-buffer &optional (eof-error-p t) eof-value) 133 (declare (type input-buffer input-buffer)) 134 (when-let ((vec (input-buffer-vector input-buffer)) 135 (pos (input-buffer-pos input-buffer))) 136 (when (< pos (length vec)) 137 (incf (input-buffer-pos input-buffer)) 138 (return-from fast-read-byte (aref vec pos)))) 139 (when-let ((stream (input-buffer-stream input-buffer))) 140 (let ((byte (read-byte stream eof-error-p eof-value))) 141 (unless (equal byte eof-value) 142 (incf (input-buffer-pos input-buffer))) 143 (return-from fast-read-byte byte))) 145 (error 'end-of-file :stream input-buffer) 148 (defun fast-peek-byte (input-buffer &optional peek-type (eof-error-p t) eof-value) 149 "This is like `peek-byte' only for fast-io input-buffers." 150 (declare (type input-buffer input-buffer)) 151 (loop :for octet = (fast-read-byte input-buffer eof-error-p :eof) 152 :for new-pos :from (input-buffer-pos input-buffer) 153 :until (cond ((eq octet :eof) 158 ((= octet peek-type))) 159 :finally (setf (buffer-position input-buffer) new-pos) 162 (defun fast-write-sequence (sequence output-buffer &optional (start 0) end) 163 (if (streamp (output-buffer-output output-buffer)) 165 (flush output-buffer) 166 (write-sequence sequence (output-buffer-output output-buffer) :start start :end end)) 168 (let* ((start2 start) 171 (- (length sequence) start))) 173 (- (length (output-buffer-vector output-buffer)) 174 (output-buffer-fill output-buffer)))) 175 (when (> buffer-remaining 0) 176 (replace (output-buffer-vector output-buffer) 177 (the octet-vector sequence) 178 :start1 (output-buffer-fill output-buffer) 181 (incf start2 buffer-remaining) 182 (incf (output-buffer-fill output-buffer) 183 (min buffer-remaining len))) 184 (let ((sequence-remaining (- (or end (length sequence)) start2))) 185 (when (> sequence-remaining 0) 186 (extend output-buffer sequence-remaining) 187 (replace (output-buffer-vector output-buffer) 188 (the octet-vector sequence) 191 (incf (output-buffer-fill output-buffer) sequence-remaining))) 192 (incf (output-buffer-len output-buffer) len) 195 (defun fast-read-sequence (sequence input-buffer &optional (start 0) end) 196 (declare (type octet-vector sequence) 197 (type input-buffer input-buffer)) 201 (- (length sequence) start)))) 202 (when-let ((vec (input-buffer-vector input-buffer)) 203 (pos (input-buffer-pos input-buffer))) 204 (when (< pos (length vec)) 205 (let ((len (min total-len (- (length vec) pos)))) 206 (replace sequence vec 210 (incf (input-buffer-pos input-buffer) len) 212 (when (< start1 total-len) 213 (when-let ((stream (input-buffer-stream input-buffer))) 214 (let ((bytes-read (read-sequence sequence stream 216 :end (+ total-len start1)))) 217 (incf (input-buffer-pos input-buffer) bytes-read) 218 (return-from fast-read-sequence bytes-read)))) 221 (defun finish-output-buffer (output-buffer) 222 "Finish an output buffer. If it is backed by a vector (static or otherwise) 223 it returns the final octet vector. If it is backed by a stream it ensures that 224 all data has been flushed to the stream." 225 (if (streamp (output-buffer-output output-buffer)) 226 (flush output-buffer) 227 (concat-buffer output-buffer))) 229 (defmacro with-fast-output ((buffer &optional output) &body body) 230 "Create `BUFFER`, optionally outputting to `OUTPUT`." 231 `(let ((,buffer (make-output-buffer :output ,output))) 233 (if (streamp (output-buffer-output ,buffer)) 235 (finish-output-buffer ,buffer)))) 237 (defmacro with-fast-input ((buffer vector &optional stream (offset 0)) &body body) 238 `(let ((,buffer (make-input-buffer :vector ,vector :stream ,stream :pos ,offset))) 242 ;;; WRITE-UNSIGNED-BE, READ-UNSIGNED-BE, etc taken from PACK, which is 243 ;;; in the public domain. 245 (defmacro write-unsigned-be (value size buffer) 246 (once-only (value buffer) 248 ,@(loop for i from (* (1- size) 8) downto 0 by 8 249 collect `(fast-write-byte (ldb (byte 8 ,i) ,value) ,buffer))))) 251 (defmacro read-unsigned-be (size buffer) 252 (with-gensyms (value) 255 ,@(loop for i from (* (1- size) 8) downto 0 by 8 256 collect `(setf (ldb (byte 8 ,i) ,value) (fast-read-byte ,buffer))) 259 (defmacro write-unsigned-le (value size buffer) 260 (once-only (value buffer) 262 ,@(loop for i from 0 below (* 8 size) by 8 263 collect `(fast-write-byte (ldb (byte 8 ,i) ,value) ,buffer))))) 265 (defmacro read-unsigned-le (size buffer) 266 (with-gensyms (value) 269 ,@(loop for i from 0 below (* 8 size) by 8 270 collect `(setf (ldb (byte 8 ,i) ,value) (fast-read-byte ,buffer))) 273 (declaim (inline unsigned-to-signed)) 274 (defun unsigned-to-signed (value size) 275 (let ((max-signed (expt 2 (1- (* 8 size)))) 276 (to-subtract (expt 2 (* 8 size)))) 277 (if (>= value max-signed) 278 (- value to-subtract) 281 (declaim (inline signed-to-unsigned)) 282 (defun signed-to-unsigned (value size) 284 (+ value (expt 2 (* 8 size))) 287 (defmacro make-readers (&rest bitlens) 288 (let ((names (mapcar (lambda (n) 289 (mapcar (lambda (m) (symbolicate (format nil m n))) 290 '("READ~A-BE" "READU~A-BE" 291 "READ~A-LE" "READU~A-LE"))) 293 `(eval-when (:compile-toplevel :load-toplevel :execute) 294 (declaim (inline ,@(flatten names))) 295 ,@(loop for fun in names 297 as bytes = (truncate bits 8) 300 (defun ,(first fun) (buffer) 301 (unsigned-to-signed (read-unsigned-be ,bytes buffer) ,bytes)) 302 (defun ,(second fun) (buffer) 303 (read-unsigned-be ,bytes buffer)) 304 (defun ,(third fun) (buffer) 305 (unsigned-to-signed (read-unsigned-le ,bytes buffer) ,bytes)) 306 (defun ,(fourth fun) (buffer) 307 (read-unsigned-le ,bytes buffer))))))) 309 (defmacro make-writers (&rest bitlens) 310 (let ((names (mapcar (lambda (n) 311 (mapcar (lambda (m) (symbolicate (format nil m n))) 312 '("WRITE~A-BE" "WRITEU~A-BE" 313 "WRITE~A-LE" "WRITEU~A-LE"))) 315 `(eval-when (:compile-toplevel :load-toplevel :execute) 316 (declaim (notinline ,@(flatten names))) 317 ,@(loop for fun in names 319 as bytes = (truncate bits 8) 322 (defun ,(first fun) (value buffer) 323 (declare (type (signed-byte ,bits) value)) 324 (write-unsigned-be (the (unsigned-byte ,bits) 325 (signed-to-unsigned value ,bytes)) ,bytes buffer)) 326 (defun ,(second fun) (value buffer) 327 (declare (type (unsigned-byte ,bits) value)) 328 (write-unsigned-be (the (unsigned-byte ,bits) value) 330 (defun ,(third fun) (value buffer) 331 (declare (type (signed-byte ,bits) value)) 332 (write-unsigned-le (the (unsigned-byte ,bits) 333 (signed-to-unsigned value ,bytes)) ,bytes buffer)) 334 (defun ,(fourth fun) (value buffer) 335 (declare (type (unsigned-byte ,bits) value)) 336 (write-unsigned-le (the (unsigned-byte ,bits) value) 339 (make-writers 16 24 32 64 128) 340 (make-readers 16 24 32 64 128) 342 (declaim (inline write8 writeu8 read8 readu8)) 343 (defun write8 (value buffer) 344 (declare (type (signed-byte 8) value)) 345 (fast-write-byte (signed-to-unsigned value 1) buffer)) 347 (defun writeu8 (value buffer) 348 (declare (type (unsigned-byte 8) value)) 349 (fast-write-byte value buffer)) 352 (defun read8 (buffer) 353 (unsigned-to-signed (fast-read-byte buffer) 1)) 355 (defun readu8 (buffer) 356 (fast-read-byte buffer)) 358 (setf (symbol-function 'write8-le) #'write8) 359 (setf (symbol-function 'write8-be) #'write8) 360 (setf (symbol-function 'writeu8-le) #'writeu8) 361 (setf (symbol-function 'writeu8-be) #'writeu8) 363 (setf (symbol-function 'read8-le) #'read8) 364 (setf (symbol-function 'read8-be) #'read8) 365 (setf (symbol-function 'readu8-le) #'readu8) 366 (setf (symbol-function 'readu8-be) #'readu8) 370 (defclass fast-io-stream (sb-gray:fundamental-stream) 371 ((openp :type boolean :initform t))) 373 (defmethod stream-file-position ((stream fast-io-stream)) 374 (with-slots (buffer) stream 375 (buffer-position buffer))) 377 (defmethod open-stream-p ((stream fast-io-stream)) 378 (slot-value stream 'openep)) 380 ;; fast-output-stream 382 (defclass fast-output-stream (fast-io-stream sb-gray:fundamental-output-stream) 383 ((buffer :type output-buffer))) 385 (defmethod initialize-instance ((self fast-output-stream) &key stream 386 buffer-size &allow-other-keys) 388 (let ((*default-output-buffer-size* (or buffer-size *default-output-buffer-size*))) 389 (with-slots (buffer) self 390 (setf buffer (make-output-buffer :output stream))))) 392 (defmethod output-stream-p ((stream fast-output-stream)) 393 (with-slots (buffer) stream 394 (and (typep buffer 'output-buffer)))) 396 (defmethod stream-element-type ((stream fast-output-stream)) 397 "Return the underlying array element-type. 398 Should always return '(unsigned-byte 8)." 399 (with-slots (buffer) stream 400 (array-element-type (output-buffer-vector buffer)))) 402 (defmethod stream-write-byte ((stream fast-output-stream) byte) 403 (with-slots (buffer) stream 404 (fast-write-byte byte buffer))) 406 (defmethod stream-write-sequence ((stream fast-output-stream) sequence start end 407 &key &allow-other-keys) 408 (with-slots (buffer) stream 409 (fast-write-sequence sequence buffer start end)) 412 (defun finish-output-stream (stream) 413 (with-slots (buffer) stream 414 (if (streamp (output-buffer-output buffer)) 416 (finish-output-buffer buffer)))) 418 (defmethod close ((stream fast-output-stream) &key abort) 419 (declare (ignore abort)) 420 (finish-output-stream stream) 421 (setf (slot-value stream 'openp) nil)) 425 (defclass fast-input-stream (fast-io-stream sb-gray:fundamental-input-stream) 426 ((buffer :type input-buffer))) 428 (defmethod initialize-instance ((self fast-input-stream) &key stream 429 vector &allow-other-keys) 431 (with-slots (buffer) self 432 (setf buffer (make-input-buffer :vector vector :stream stream)))) 434 (defmethod input-stream-p ((stream fast-input-stream)) 435 (with-slots (buffer) stream 436 (and (typep buffer 'input-buffer)))) 438 (defmethod stream-element-type ((stream fast-input-stream)) 439 "Return element-type of the underlying vector or stream. 440 Return NIL if none are present." 441 (with-slots (buffer) stream 442 (if-let ((vec (input-buffer-vector buffer))) 443 (array-element-type vec) 444 (when-let ((stream (input-buffer-stream buffer))) 445 (stream-element-type stream))))) 447 (defmethod (setf stream-file-position) (new-pos (stream fast-input-stream)) 448 (with-slots (buffer) stream 449 (setf (buffer-position buffer) new-pos))) 451 (defmethod peek-byte ((stream fast-input-stream) &optional peek-type (eof-error-p t) eof-value) 452 (with-slots (buffer) stream 453 (fast-peek-byte buffer peek-type eof-error-p eof-value))) 455 (defmethod stream-read-byte ((stream fast-input-stream)) 456 (with-slots (buffer) stream 457 (fast-read-byte buffer))) 459 (defmethod stream-read-sequence ((stream fast-input-stream) sequence start end 460 &key &allow-other-keys) 461 (with-slots (buffer) stream 462 (fast-read-sequence sequence buffer start end))) 464 (defmethod close ((stream fast-input-stream) &key abort) 465 (declare (ignore abort)) 466 (setf (slot-value stream 'openp) nil))