1.1--- a/lisp/lib/net/fetch.lisp Wed May 22 22:16:26 2024 -0400
1.2+++ b/lisp/lib/net/fetch.lisp Wed May 22 22:46:17 2024 -0400
1.3@@ -8,7 +8,7 @@
1.4 output
1.5 (file-namestring (obj/uri:uri-path (obj/uri:uri url))))))
1.6 (multiple-value-bind (stream status header uri)
1.7- (dex:get url :want-stream t)
1.8+ (req:get url :want-stream t)
1.9 (when (= status 200) (write-stream-into-file stream (pathname output)))
1.10 (values (or stream uri header)
1.11 status))))
2.1--- a/lisp/lib/net/pkg.lisp Wed May 22 22:16:26 2024 -0400
2.2+++ b/lisp/lib/net/pkg.lisp Wed May 22 22:46:17 2024 -0400
2.3@@ -156,6 +156,7 @@
2.4 (:export))
2.5
2.6 (defpackage :net/proto/http
2.7+ (:nicknames :http)
2.8 (:use :cl :std :net/core :sb-bsd-sockets :parse/bytes :io/xsubseq :io/smart-buffer)
2.9 (:export
2.10 :make-parser
2.11@@ -274,10 +275,10 @@
2.12
2.13 (defpackage :net/req
2.14 (:nicknames :req)
2.15- (:shadowing-import-from :babel :string-to-octets)
2.16 (:shadowing-import-from :std/type :octet :octet-vector)
2.17 (:shadow :get :delete)
2.18- (:use :cl :std :obj/uri :net/proto/http :sb-ext :babel :net/cookie :fast-io :dat/base64)
2.19+ (:use :cl :std :obj/uri :net/proto/http :babel :net/cookie :fast-io :dat/base64 :cl+ssl :sb-gray)
2.20+ (:shadowing-import-from :babel :octets-to-string)
2.21 (:export
2.22 :request
2.23 :get
3.1--- a/lisp/lib/net/proto/http.lisp Wed May 22 22:16:26 2024 -0400
3.2+++ b/lisp/lib/net/proto/http.lisp Wed May 22 22:46:17 2024 -0400
3.3@@ -181,7 +181,7 @@
3.4 (setq dot-read-p t))
3.5 (T (return-from number-string-p nil))))))))
3.6
3.7-;;; fast-http
3.8+;;; http
3.9 (defun make-parser (http &key first-line-callback header-callback body-callback finish-callback (head-request nil))
3.10 (declare (type http http))
3.11 (let (callbacks
3.12@@ -223,12 +223,12 @@
3.13 header-complete-p nil
3.14 completedp nil))
3.15 :url (lambda (http data start end)
3.16- (declare (type simple-byte-vector data)
3.17+ (declare (type octet-vector data)
3.18 (type pointer start end))
3.19 (setf (http-resource http)
3.20 (ascii-octets-to-string data :start start :end end)))
3.21 :status (lambda (http data start end)
3.22- (declare (type simple-byte-vector data)
3.23+ (declare (type octet-vector data)
3.24 (type pointer start end))
3.25 (setf (http-status-text http)
3.26 (ascii-octets-to-string data :start start :end end)))
3.27@@ -238,7 +238,7 @@
3.28 (funcall (the function first-line-callback))))
3.29 :header-field (lambda (http data start end)
3.30 (declare (ignore http)
3.31- (type simple-byte-vector data)
3.32+ (type octet-vector data)
3.33 (type pointer start end))
3.34 (collect-prev-header-value)
3.35 (setq header-value-buffer (make-concatenated-xsubseqs))
3.36@@ -246,7 +246,7 @@
3.37 (ascii-octets-to-lower-string data :start start :end end)))
3.38 :header-value (lambda (http data start end)
3.39 (declare (ignore http)
3.40- (type simple-byte-vector data)
3.41+ (type octet-vector data)
3.42 (type pointer start end))
3.43 (xnconcf header-value-buffer
3.44 (xsubseq (subseq (the octet-vector data) start end) 0)))
3.45@@ -266,7 +266,7 @@
3.46 :body (and body-callback
3.47 (lambda (http data start end)
3.48 (declare (ignore http)
3.49- (type simple-byte-vector data)
3.50+ (type octet-vector data)
3.51 (type pointer start end))
3.52 (funcall (the function body-callback)
3.53 data start end)))
3.54@@ -285,20 +285,20 @@
3.55 (when finish-callback
3.56 (funcall (the function finish-callback))))
3.57 (T
3.58- (locally (declare (type simple-byte-vector data)
3.59+ (locally (declare (type octet-vector data)
3.60 (type pointer start))
3.61 (check-type end (or null pointer))
3.62 (when data-buffer
3.63 (setq data
3.64 (coerce 'list
3.65 (xnconc (xsubseq data-buffer 0)
3.66- (xsubseq (the simple-byte-vector data) start (or end (length data))))))
3.67+ (xsubseq (the octet-vector data) start (or end (length data))))))
3.68 (setq data-buffer nil
3.69 start 0
3.70 end nil))
3.71 (setf (http-mark http) start)
3.72 (handler-case
3.73- (funcall parse-fn http callbacks (the simple-byte-vector data) :start start :end end :head-request head-request)
3.74+ (funcall parse-fn http callbacks (the octet-vector data) :start start :end end :head-request head-request)
3.75 (eof ()
3.76 (setq data-buffer
3.77 (subseq data (http-mark http) (or end (length data)))))))))
3.78@@ -336,7 +336,7 @@
3.79 :initial-contents (list +cr+ +lf+))
3.80 :test 'equalp)
3.81
3.82-(deftype simple-byte-vector (&optional (len '*))
3.83+(deftype octet-vector (&optional (len '*))
3.84 `(simple-array (unsigned-byte 8) (,len)))
3.85
3.86 (declaim (inline digit-byte-char-p
3.87@@ -402,7 +402,7 @@
3.88
3.89 (declaim (inline ascii-octets-to-string))
3.90 (defun ascii-octets-to-string (octets &key (start 0) (end (length octets)))
3.91- (declare (type simple-byte-vector octets)
3.92+ (declare (type octet-vector octets)
3.93 (type (unsigned-byte 64) start end)
3.94 (optimize (speed 3) (safety 0)))
3.95 (let* ((len (the (unsigned-byte 64) (- end start)))
3.96@@ -417,7 +417,7 @@
3.97
3.98 (declaim (inline ascii-octets-to-lower-string))
3.99 (defun ascii-octets-to-lower-string (octets &key (start 0) (end (length octets)))
3.100- (declare (type simple-byte-vector octets)
3.101+ (declare (type octet-vector octets)
3.102 (type (unsigned-byte 64) start end)
3.103 (optimize (speed 3) (safety 0)))
3.104 (let* ((len (the (unsigned-byte 64) (- end start)))
3.105@@ -431,13 +431,13 @@
3.106 (code-char (byte-to-ascii-lower (aref octets j)))))))
3.107
3.108 (defun append-byte-vectors (vec1 vec2)
3.109- (declare (type simple-byte-vector vec1 vec2)
3.110+ (declare (type octet-vector vec1 vec2)
3.111 (optimize (speed 3) (safety 0)))
3.112 (let* ((vec1-len (length vec1))
3.113 (vec2-len (length vec2))
3.114 (result (make-array (+ vec1-len vec2-len)
3.115 :element-type '(unsigned-byte 8))))
3.116- (declare (type simple-byte-vector result))
3.117+ (declare (type octet-vector result))
3.118 (replace result vec1 :start1 0)
3.119 (replace result vec2 :start1 vec1-len)
3.120 result))
3.121@@ -476,12 +476,12 @@
3.122 collect `(defconstant ,(format-symbol t "+~A+" state) ,i)))
3.123
3.124 (defun http-multipart-parse (parser callbacks data &key (start 0) end)
3.125- (declare (type simple-byte-vector data))
3.126+ (declare (type octet-vector data))
3.127 (let* ((end (or end (length data)))
3.128 (boundary (map '(simple-array (unsigned-byte 8) (*)) #'char-code (ll-multipart-parser-boundary parser)))
3.129 (boundary-length (length boundary))
3.130 (header-parser (ll-multipart-parser-header-parser parser)))
3.131- (declare (type simple-byte-vector boundary))
3.132+ (declare (type octet-vector boundary))
3.133 (when (= start end)
3.134 (return-from http-multipart-parse start))
3.135
3.136@@ -547,7 +547,7 @@
3.137 ((ll-multipart-parser-boundary-buffer parser)
3.138 (when (< (+ end (length (ll-multipart-parser-boundary-buffer parser)) -3) end2)
3.139 (setf (ll-multipart-parser-boundary-buffer parser)
3.140- (concatenate 'simple-byte-vector
3.141+ (concatenate 'octet-vector
3.142 (ll-multipart-parser-boundary-buffer parser)
3.143 data))
3.144 (go exit-loop))
3.145@@ -569,7 +569,7 @@
3.146 ;; EOF
3.147 (setf (ll-multipart-parser-boundary-buffer parser)
3.148 (if (ll-multipart-parser-boundary-buffer parser)
3.149- (concatenate 'simple-byte-vector
3.150+ (concatenate 'octet-vector
3.151 (ll-multipart-parser-boundary-buffer parser)
3.152 (subseq data (max 0 (- p 2))))
3.153 (subseq data (max 0 (- p 2)))))
3.154@@ -684,7 +684,7 @@
3.155 (when (ll-multipart-parser-boundary-mark parser)
3.156 (setf (ll-multipart-parser-body-buffer parser)
3.157 (if (ll-multipart-parser-body-buffer parser)
3.158- (concatenate 'simple-byte-vector
3.159+ (concatenate 'octet-vector
3.160 (ll-multipart-parser-body-buffer parser)
3.161 (subseq data (ll-multipart-parser-boundary-mark parser)))
3.162 (subseq data (ll-multipart-parser-boundary-mark parser)))))
3.163@@ -1055,7 +1055,7 @@
3.164 ;; Main
3.165
3.166 (defun parse-method (data start end)
3.167- (declare (type simple-byte-vector data)
3.168+ (declare (type octet-vector data)
3.169 (type pointer start end))
3.170 (with-octets-parsing (data :start start :end end)
3.171 (return-from parse-method
3.172@@ -1096,7 +1096,7 @@
3.173 (error 'eof))
3.174
3.175 (defun parse-url (data start end)
3.176- (declare (type simple-byte-vector data)
3.177+ (declare (type octet-vector data)
3.178 (type pointer start end))
3.179 (flet ((url-char-byte-p (byte)
3.180 (or (<= (char-code #\!) byte (char-code #\~))
3.181@@ -1107,7 +1107,7 @@
3.182 (error 'eof)))
3.183
3.184 (defun parse-http-version (data start end)
3.185- (declare (type simple-byte-vector data)
3.186+ (declare (type octet-vector data)
3.187 (type pointer start end))
3.188 (let (major minor)
3.189 (with-octets-parsing (data :start start :end end)
3.190@@ -1127,7 +1127,7 @@
3.191 (error 'eof)))
3.192
3.193 (defun parse-status-code (http callbacks data start end)
3.194- (declare (type simple-byte-vector data)
3.195+ (declare (type octet-vector data)
3.196 (type pointer start end))
3.197 (or (with-octets-parsing (data :start start :end end)
3.198 (if (digit-byte-char-p (current))
3.199@@ -1161,7 +1161,7 @@
3.200 (error 'eof)))
3.201
3.202 (defun parse-header-field-and-value (http callbacks data start end)
3.203- (declare (type simple-byte-vector data)
3.204+ (declare (type octet-vector data)
3.205 (type pointer start end))
3.206 (or
3.207 (with-octets-parsing (data :start start :end end)
3.208@@ -1262,7 +1262,7 @@
3.209 (error 'eof)))
3.210
3.211 (defun parse-header-value-transfer-encoding (data start end)
3.212- (declare (type simple-byte-vector data)
3.213+ (declare (type octet-vector data)
3.214 (type pointer start end))
3.215 (with-octets-parsing (data :start start :end end)
3.216 (match-i-case
3.217@@ -1288,7 +1288,7 @@
3.218 (error 'eof))
3.219
3.220 (defun parse-header-value-content-length (data start end)
3.221- (declare (type simple-byte-vector data)
3.222+ (declare (type octet-vector data)
3.223 (type pointer start end))
3.224 (let ((content-length 0))
3.225 (declare (type integer content-length))
3.226@@ -1315,7 +1315,7 @@
3.227 (error 'eof)))
3.228
3.229 (defun parse-header-line (http callbacks data start end)
3.230- (declare (type simple-byte-vector data)
3.231+ (declare (type octet-vector data)
3.232 (type pointer start end))
3.233 (when (<= end start)
3.234 (error 'eof))
3.235@@ -1338,7 +1338,7 @@
3.236
3.237 (defun parse-headers (http callbacks data start end)
3.238 (declare (type http http)
3.239- (type simple-byte-vector data)
3.240+ (type octet-vector data)
3.241 (type pointer start end))
3.242 (or (with-octets-parsing (data :start start :end end)
3.243 ;; empty headers
3.244@@ -1368,7 +1368,7 @@
3.245
3.246 (defun read-body-data (http callbacks data start end)
3.247 (declare (type http http)
3.248- (type simple-byte-vector data)
3.249+ (type octet-vector data)
3.250 (type pointer start end))
3.251 (let ((readable-count (the pointer (- end start))))
3.252 (declare (dynamic-extent readable-count)
3.253@@ -1405,7 +1405,7 @@
3.254
3.255 (defun parse-body (http callbacks data start end requestp)
3.256 (declare (type http http)
3.257- (type simple-byte-vector data)
3.258+ (type octet-vector data)
3.259 (type pointer start end))
3.260 (macrolet ((message-complete ()
3.261 `(progn
3.262@@ -1444,7 +1444,7 @@
3.263
3.264 (defun parse-chunked-body (http callbacks data start end)
3.265 (declare (type http http)
3.266- (type simple-byte-vector data)
3.267+ (type octet-vector data)
3.268 (type pointer start end))
3.269
3.270 (when (= start end)
3.271@@ -1535,7 +1535,7 @@
3.272
3.273 (defun parse-request (http callbacks data &key (start 0) end (head-request nil))
3.274 (declare (type http http)
3.275- (type simple-byte-vector data)
3.276+ (type octet-vector data)
3.277 (ignorable head-request))
3.278 (let ((end (or end (length data))))
3.279 (declare (type pointer start end))
3.280@@ -1641,7 +1641,7 @@
3.281
3.282 (defun parse-response (http callbacks data &key (start 0) end (head-request nil))
3.283 (declare (type http http)
3.284- (type simple-byte-vector data))
3.285+ (type octet-vector data))
3.286 (let ((end (or end
3.287 (length data))))
3.288 (declare (type pointer start end))
4.1--- a/lisp/lib/net/req.lisp Wed May 22 22:16:26 2024 -0400
4.2+++ b/lisp/lib/net/req.lisp Wed May 22 22:46:17 2024 -0400
4.3@@ -280,7 +280,7 @@
4.4 (main 0)))
4.5
4.6 ;;; keep-alive-stream
4.7-(defclass keep-alive-stream (fundamental-input-stream)
4.8+(defclass keep-alive-stream (sb-gray:fundamental-input-stream)
4.9 ((stream :type (or null stream)
4.10 :initarg :stream
4.11 :initform (error ":stream is required")
4.12@@ -347,7 +347,7 @@
4.13 byte))
4.14 (or (maybe-close stream t) :eof))))
4.15
4.16-(defmethod stream-read-sequence ((stream keep-alive-stream) sequence start end &key)
4.17+(defmethod stream-read-sequence ((stream keep-alive-stream) sequence &optional start end)
4.18 (declare (optimize speed))
4.19 (if (null (keep-alive-stream-stream stream)) ;; we already closed it
4.20 start
4.21@@ -359,7 +359,7 @@
4.22 (maybe-close stream (<= (keep-alive-stream-end stream) 0))
4.23 n)))
4.24
4.25-(defmethod stream-read-sequence ((stream keep-alive-chunked-stream) sequence start end &key)
4.26+(defmethod stream-read-sequence ((stream keep-alive-chunked-stream) sequence &optional start end)
4.27 (declare (optimize speed))
4.28 (if (null (keep-alive-stream-stream stream)) ;; we already closed it
4.29 start
4.30@@ -616,7 +616,7 @@
4.31 ((array (unsigned-byte 8) (*)) (write-sequence val stream))
4.32 (pathname
4.33 (with-open-file (in val :element-type '(unsigned-byte 8))
4.34- (copy-stream in stream)))
4.35+ (std/stream:copy-stream in stream)))
4.36 (string
4.37 (write-sequence (convert-to-octets val) stream))
4.38 (cons (write-as-octets stream (first val)))
4.39@@ -1678,7 +1678,7 @@
4.40 :if-exists if-exists
4.41 :if-does-not-exist :create)
4.42 (remf args :if-exists)
4.43- (let ((body (apply #'dex:get uri :want-stream t :force-binary t
4.44+ (let ((body (apply #'req:get uri :want-stream t :force-binary t
4.45 args)))
4.46 (alexandria:copy-stream body out)
4.47 ;; Nominally the body gets closed, but if keep-alive is nil we need to explicitly do it.
5.1--- a/lisp/lib/parse/bytes.lisp Wed May 22 22:16:26 2024 -0400
5.2+++ b/lisp/lib/parse/bytes.lisp Wed May 22 22:46:17 2024 -0400
5.3@@ -177,15 +177,12 @@
5.4 (eq (car var) 'the)
5.5 (cadr var)))))
5.6
5.7-(deftype octets (&optional (len '*))
5.8- `(simple-array (unsigned-byte 8) (,len)))
5.9-
5.10 (defun variable-type* (var &optional env)
5.11 (let ((type (variable-type var env)))
5.12 (cond
5.13 ((null type) nil)
5.14 ((subtypep type 'string) 'string)
5.15- ((subtypep type 'octets) 'octets))))
5.16+ ((subtypep type 'octet-vector) 'octet-vector))))
5.17
5.18 (defun check-skip-elems (elems)
5.19 (or (every (lambda (elem)
5.20@@ -406,7 +403,7 @@
5.21 (,g-end ,(if end
5.22 `(or ,end (length ,data))
5.23 `(length ,data))))
5.24- (declare (type octets ,data)
5.25+ (declare (type octet-vector ,data)
5.26 (type fixnum ,p ,g-end)
5.27 (type (unsigned-byte 8) ,elem))
5.28 (parsing-macrolet (,elem ,data ,p ,g-end)
5.29@@ -469,14 +466,14 @@
5.30 (let ((data-type (variable-type* data env)))
5.31 (case data-type
5.32 (string `(with-string-parsing (,data :start ,start :end ,end) ,@body))
5.33- (octets `(macrolet ((get-elem (form) `(code-char ,form))
5.34+ (octet-vector `(macrolet ((get-elem (form) `(code-char ,form))
5.35 (subseq* (data start &optional end)
5.36 `(babel:octets-to-string ,data :start ,start :end ,end)))
5.37 (with-octets-parsing (,data :start ,start :end ,end) ,@body)))
5.38 (otherwise (once-only (data)
5.39 `(etypecase ,data
5.40 (string (with-string-parsing (,data :start ,start :end ,end) ,@body))
5.41- (octets (macrolet ((get-elem (form) `(code-char ,form))
5.42+ (octet-vector (macrolet ((get-elem (form) `(code-char ,form))
5.43 (subseq* (data start &optional end)
5.44 `(babel:octets-to-string ,data :start ,start :end ,end)))
5.45 (with-octets-parsing (,data :start ,start :end ,end) ,@body)))))))))
6.1--- a/lisp/lib/parse/pkg.lisp Wed May 22 22:16:26 2024 -0400
6.2+++ b/lisp/lib/parse/pkg.lisp Wed May 22 22:46:17 2024 -0400
6.3@@ -39,7 +39,7 @@
6.4 (:import-from :sb-cltl2
6.5 :variable-information)
6.6 (:import-from :std :with-gensyms :once-only
6.7- :ensure-cons :ignore-some-conditions)
6.8+ :ensure-cons :ignore-some-conditions :octet-vector :octet)
6.9 (:export :with-vector-parsing
6.10 :with-string-parsing
6.11 :with-octets-parsing
