1.1--- a/lisp/lib/dat/parquet/gen.lisp Mon Jul 15 20:59:04 2024 -0400
1.2+++ b/lisp/lib/dat/parquet/gen.lisp Mon Jul 15 21:59:45 2024 -0400
1.3@@ -91,7 +91,8 @@
1.4 (def-parquet-enum page-types "PageType")
1.5 (def-parquet-enum boundary-orders "BoundaryOrder")))
1.6
1.7-(defvar *parquet-structs* nil)
1.8+(eval-always
1.9+ (defvar *parquet-structs* nil))
1.10 (defstruct (parquet-struct
1.11 (:constructor make-parquet-struct (name doc exceptionp unionp fields)))
1.12 name doc exceptionp unionp (fields nil :type list))
2.1--- a/lisp/lib/dat/parquet/tcompact.lisp Mon Jul 15 20:59:04 2024 -0400
2.2+++ b/lisp/lib/dat/parquet/tcompact.lisp Mon Jul 15 21:59:45 2024 -0400
2.3@@ -133,7 +133,7 @@
2.4 (defun tcompact-encode-field-header (field)
2.5 (concatenate 'octet-vector
2.6 (tcompact-encode-field-header-short 0 (tcompact-field-type-id field))
2.7- (tocmpact-encode-field-id (tcompact-field-id field))))
2.8+ (tcompact-encode-field-id (tcompact-field-id field))))
2.9
2.10 (defun tcompact-encode-field-value (field))
2.11
3.1--- a/lisp/lib/dat/tests.lisp Mon Jul 15 20:59:04 2024 -0400
3.2+++ b/lisp/lib/dat/tests.lisp Mon Jul 15 21:59:45 2024 -0400
3.3@@ -146,12 +146,17 @@
3.4 (with-output-to-string (s)
3.5 (is (write-sxp-stream f s)))))
3.6
3.7+(defparameter *parquet-test-file*
3.8+ (probe-file
3.9+ (merge-pathnames "../../../.stash/datapage_v1-uncompressed-checksum.parquet"
3.10+ #.(asdf:system-source-directory :dat/tests))))
3.11 ;; see also: https://github.com/apache/parquet-testing/blob/master/data/README.md
3.12 (deftest parquet-basic ()
3.13- (with-open-file (st "/home/ellis/comp/core/.stash/datapage_v1-uncompressed-checksum.parquet" :element-type '(unsigned-byte 8))
3.14- (let ((footer (dat/parquet::parquet-read-footer st)))
3.15- (is (typep footer
3.16- 'dat/parquet::parquet-file-meta-data))
3.17- (print (slot-value footer 'dat/parquet::schema))
3.18- (print (file-position st))
3.19- (print (file-length st)))))
3.20+ (when *parquet-test-file*
3.21+ (with-open-file (st *parquet-test-file* :element-type '(unsigned-byte 8))
3.22+ (let ((footer (dat/parquet::parquet-read-footer st)))
3.23+ (is (typep footer
3.24+ 'dat/parquet::parquet-file-meta-data))
3.25+ (print (slot-value footer 'dat/parquet::schema))
3.26+ (print (file-position st))
3.27+ (print (file-length st))))))
4.1--- a/lisp/lib/net/fetch.lisp Mon Jul 15 20:59:04 2024 -0400
4.2+++ b/lisp/lib/net/fetch.lisp Mon Jul 15 21:59:45 2024 -0400
4.3@@ -8,8 +8,12 @@
4.4 output
4.5 (file-namestring (obj/uri:uri-path (obj/uri:uri url))))))
4.6 (multiple-value-bind (stream status header uri)
4.7- (req:get url :want-stream t :keep-alive nil :use-connection-pool t :force-binary t)
4.8- (when (= status 200) (write-stream-into-file stream (pathname output) :if-exists if-exists))
4.9+ (req:get url :want-stream t :force-binary t)
4.10+ (when (= status 200)
4.11+ (with-open-file (out output :direction :output :element-type '(unsigned-byte 8) :if-exists if-exists)
4.12+ (loop for c = (read-byte stream nil nil)
4.13+ while c
4.14+ do (write-byte c out))))
4.15 (values (or stream uri header)
4.16 status))))
4.17
4.18@@ -47,7 +51,7 @@
4.19 (when (is-file file-pathname) (delete-file file-pathname))
4.20 (if (and cache (probe-file file-pathname))
4.21 (values file-pathname 200 "OK")
4.22- (download url-or-path file-pathname)))))
4.23+ (download url-or-path :output file-pathname)))))
4.24 (t (values nil 404 "Not file of url"))))
4.25
4.26 (defun fetch (url-or-path
5.1--- a/lisp/std/file.lisp Mon Jul 15 20:59:04 2024 -0400
5.2+++ b/lisp/std/file.lisp Mon Jul 15 21:59:45 2024 -0400
5.3@@ -310,13 +310,12 @@
5.4 STREAM will be closed afterwards, so wrap it with
5.5 `make-concatenated-stream' if you want it left open."
5.6 (check-type pathname pathname)
5.7- (with-open-stream (in stream)
5.8- (with-output-to-file (out pathname
5.9- :element-type (stream-element-type in)
5.10- :if-exists if-exists
5.11- :if-does-not-exist if-does-not-exist)
5.12- (copy-stream in out)))
5.13- pathname)
5.14+ (with-output-to-file (out pathname
5.15+ :element-type (stream-element-type stream)
5.16+ :if-exists if-exists
5.17+ :if-does-not-exist if-does-not-exist)
5.18+ (copy-stream stream out))
5.19+pathname)
5.20
5.21 (defun write-file-into-stream (pathname output &key (if-does-not-exist :error)
5.22 (external-format :default))
6.1--- a/skelfile Mon Jul 15 20:59:04 2024 -0400
6.2+++ b/skelfile Mon Jul 15 21:59:45 2024 -0400
6.3@@ -29,22 +29,16 @@
6.4 clang -g -O2 -Wall -Wno-unused-value -ltree-sitter -shared \
6.5 alien.c -o ../../../.stash/libtree-sitter-alien.so$#)
6.6 (tree-sitter-alien-install () #$cp .stash/libtree-sitter-alien.so /usr/local/lib/$#)
6.7- (psl.dat () ;; req:fetch works in slime and sk-shell but not here? hmm..
6.8- (with-open-file
6.9- (st ".stash/psl.dat" :direction :output :element-type 'octet)
6.10- (loop for x across (req:get "https://publicsuffix.org/list/public_suffix_list.dat" :force-binary t)
6.11- do (write-byte x st))))
6.12+ (psl.dat ()
6.13+ (net/fetch:download "https://publicsuffix.org/list/public_suffix_list.dat" :output ".stash/psl.dat"))
6.14 (parquet.thrift ()
6.15- (with-open-file
6.16- (st ".stash/parquet.thrift" :direction :output :element-type 'octet)
6.17- (loop for x across (req:get "https://raw.githubusercontent.com/apache/parquet-format/master/src/main/thrift/parquet.thrift" :force-binary t)
6.18- do (write-byte x st)))
6.19+ (net/fetch:download
6.20+ "https://raw.githubusercontent.com/apache/parquet-format/master/src/main/thrift/parquet.thrift"
6.21+ :output ".stash/parquet.thrift")
6.22 #$thrift --gen json -out .stash .stash/parquet.thrift$#)
6.23 (parquet.json ()
6.24- (with-open-file
6.25- (st ".stash/parquet.json" :direction :output :element-type 'octet)
6.26- (loop for x across (req:get "https://packy.compiler.company/data/parquet.json" :force-binary t)
6.27- do (write-byte x st))))
6.28+ (net/fetch:download "https://packy.compiler.company/data/parquet.json"
6.29+ :output ".stash/parquet.json"))
6.30 ;; lisp
6.31 (%stash () #$mkdir -pv .stash$#)
6.32 (rdb (%stash) (with-sbcl (:noinform t :quit t)
