1.1--- a/emacs/default.el Sun Sep 01 21:10:08 2024 -0400
1.2+++ b/emacs/default.el Mon Sep 02 18:31:19 2024 -0400
1.3@@ -2,6 +2,7 @@
1.4
1.5 ;;; Code:
1.6 ;;; Settings
1.7+(require 'util)
1.8 (put 'upcase-region 'disabled nil)
1.9 (put 'list-threads 'disabled nil)
1.10 (put 'list-timers 'disabled nil)
1.11@@ -118,6 +119,8 @@
1.12 "DEV" "DEV_ID" "DEV_HOME"
1.13 "WORKER" "WORKER_ID" "WORKER_HOME"
1.14 "SBCL_HOME"
1.15+ "STASH"
1.16+ "STORE"
1.17 "LISP_HOME"))
1.18
1.19 (add-to-list 'exec-path (expand-file-name "~/.cargo/bin/"))
2.1--- a/emacs/early-init.el Sun Sep 01 21:10:08 2024 -0400
2.2+++ b/emacs/early-init.el Mon Sep 02 18:31:19 2024 -0400
2.3@@ -24,9 +24,6 @@
2.4 ;; startup, such as UI elements that would otherwise be loaded for a
2.5 ;; second and then disappear when the `init.el` file is loaded.
2.6 ;;
2.7-;; If any of the below settings are in your `init.el`, I suggest
2.8-;; migrating them to `early-init.el`.
2.9-;;
2.10 ;;; Code:
2.11 (setopt inhibit-startup-buffer-menu nil
2.12 initial-buffer-choice t
3.1--- a/emacs/init.el Sun Sep 01 21:10:08 2024 -0400
3.2+++ b/emacs/init.el Mon Sep 02 18:31:19 2024 -0400
3.3@@ -5,7 +5,7 @@
3.4 ;;; Code:
3.5 (add-to-list 'load-path (expand-file-name "lib" user-emacs-directory))
3.6
3.7-(dolist (x '("util.el" "default.el"))
3.8+(dolist (x '("util.el" "default.el" "keys.el"))
3.9 (let ((y (concat user-emacs-directory x)))
3.10 (if (and (native-comp-available-p) (not (eq system-type 'darwin)))
3.11 (native-compile y)
3.12@@ -18,3 +18,21 @@
3.13 (load-file user-custom-file)))
3.14
3.15 (add-hook 'after-init-hook 'load-default-theme)
3.16+
3.17+(custom-set-variables
3.18+ ;; custom-set-variables was added by Custom.
3.19+ ;; If you edit it by hand, you could mess it up, so be careful.
3.20+ ;; Your init file should contain only one such instance.
3.21+ ;; If there is more than one, they won't work right.
3.22+ '(custom-safe-themes
3.23+ '("c3e62e14eb625e02e5aeb03d315180d5bb6627785e48f23ba35eb7b974a940af"
3.24+ "01cad03be8c042a9941fda5a484280629ee2cc83fe084af6d19376c83141c91b"
3.25+ "79ab8329f4522beaa2285888d38f6204bb60f324912660d774a412a79e336d6c"
3.26+ "9ddb83c12595e789e9abd04a5c0705661748776223a794a6f64669352b956e79"
3.27+ "29a073e66535bad18e11e9bcaa17d7f2d17e4c79f01023e59e9841633915c232"
3.28+ "587ce9a1a961792114991fd488ef9c3fc37f165f6fea8b89d155640e81d165a3"
3.29+ "8a3d04fd24afde8333c1437a3ecaa616f121554041a4e7e48f21b28f13b50246"
3.30+ "4f03e70554a58349740973c69e73aefd8ce761a77b22a9dc52a19e708532084a"
3.31+ "0a953c81f5798aa99cafbc4aa8a56d16827442400028f6c1eab0c43061ea331c"
3.32+ "b93039071f490613499b76c237c2624ae67a9aafbc717da9b4d81f456344e56e"
3.33+ default)))
4.1--- a/emacs/keys.el Sun Sep 01 21:10:08 2024 -0400
4.2+++ b/emacs/keys.el Mon Sep 02 18:31:19 2024 -0400
4.3@@ -39,7 +39,7 @@
4.4 "O" #'allout-mode
4.5 "R" #'global-auto-revert-mode
4.6 "t" #'toggle-frame-tab-bar
4.7- "T" #'toggle-theme
4.8+ "T" #'load-theme
4.9 "d" #'toggle-debug-on-error
4.10 "SPC" #'toggle-macro-recording
4.11 "x" #'org-clock-toggle-auto-clockout
4.12@@ -95,6 +95,7 @@
4.13 :doc "User-specified keymap usually bound to 'C-c c' and populated in 'custom.el'."
4.14 :prefix 'user-map
4.15 "c" #'org-capture
4.16+ "1" #'org-inbox-open
4.17 "l" #'org-store-link
4.18 "C-l" #'org-web-tools-insert-link-for-url
4.19 "f" #'load-file
5.1--- a/lisp/lib/dat/dat.asd Sun Sep 01 21:10:08 2024 -0400
5.2+++ b/lisp/lib/dat/dat.asd Mon Sep 02 18:31:19 2024 -0400
5.3@@ -14,6 +14,7 @@
5.4 ((:file "gen")
5.5 (:file "pkg")
5.6 (:file "obj")
5.7+ (:file "thrift")
5.8 (:file "tcompact")
5.9 (:file "io")
5.10 (:file "rle")
6.1--- a/lisp/lib/dat/parquet/gen.lisp Sun Sep 01 21:10:08 2024 -0400
6.2+++ b/lisp/lib/dat/parquet/gen.lisp Mon Sep 02 18:31:19 2024 -0400
6.3@@ -8,15 +8,18 @@
6.4 (:export :load-parquet))
6.5
6.6 (in-package :dat/parquet/gen)
6.7+
6.8 (defparameter *parquet-json-file*
6.9- (probe-file #.(asdf:system-relative-pathname :prelude #P"../.stash/parquet.json")))
6.10+ (or (probe-file #.(asdf:system-relative-pathname :prelude #P"../.stash/parquet.json"))
6.11+ (warn "*PARQUET-JSON-FILE* not found")))
6.12+
6.13+(defparameter *parquet-output-file*
6.14+ #.(asdf:system-relative-pathname :dat #P"parquet/thrift.lisp"))
6.15+
6.16 (defvar *parquet-json* nil)
6.17-(defun load-parquet-json (&optional (json-file *parquet-json-file*))
6.18- (with-open-file (file json-file)
6.19- (setq *parquet-json* (json-read file))))
6.20
6.21 (defun %parquet-json-enums ()
6.22- (json-getf *parquet-json* "enums"))
6.23+ (json-getf *parquet-json* "enums"))
6.24
6.25 (defun parquet-json-enum-getf (name)
6.26 (json-getf
6.27@@ -25,7 +28,7 @@
6.28
6.29 (defvar *parquet-enums* nil)
6.30
6.31-(defmacro def-parquet-enum (sym name)
6.32+(defmacro define-parquet-enum (sym name)
6.33 `(progn
6.34 (defun ,(symbolicate "PARQUET-JSON-" sym) ()
6.35 (mapcar (lambda (x) (keywordicate (snakecase-name-to-lisp-name (json-getf x "name"))))
6.36@@ -99,18 +102,8 @@
6.37 `(or null ,ret)
6.38 ret)))))
6.39
6.40-(defun parquet-json-enums ()
6.41- (list
6.42- (def-parquet-enum types "Type")
6.43- (def-parquet-enum converted-types "ConvertedType")
6.44- (def-parquet-enum field-repetition-types "FieldRepetitionType")
6.45- (def-parquet-enum encodings "Encoding")
6.46- (def-parquet-enum compression-codecs "CompressionCodec")
6.47- (def-parquet-enum page-types "PageType")
6.48- (def-parquet-enum boundary-orders "BoundaryOrder")))
6.49+(defparameter *parquet-structs* nil)
6.50
6.51-(eval-always
6.52- (defvar *parquet-structs* nil))
6.53 (defstruct (parquet-struct
6.54 (:constructor make-parquet-struct (name doc exceptionp unionp fields)))
6.55 name doc exceptionp unionp (fields nil :type list))
6.56@@ -142,10 +135,12 @@
6.57 (defun parquet-json-namespaces ()
6.58 (json-getf *parquet-json* "namespaces"))
6.59
6.60-(defun init-parquet-json (&optional (file *parquet-json-file*))
6.61- (load-parquet-json file)
6.62- (setq *parquet-enums* (parquet-json-enums))
6.63- (setq *parquet-structs* (parquet-json-structs)))
6.64+(eval-always
6.65+ (defun init-parquet-json (&optional (file *parquet-json-file*))
6.66+ (with-open-file (file file)
6.67+ (setq *parquet-json* (json-read file)))
6.68+ (setq *parquet-enums* (parquet-json-enums))
6.69+ (setq *parquet-structs* (parquet-json-structs))))
6.70
6.71 ;;; CLOS
6.72
6.73@@ -175,48 +170,68 @@
6.74 (camelcase-name-to-lisp-name name))))
6.75 :dat/parquet)))
6.76 `(progn
6.77- (define-parquet-class ,class-name
6.78- (dat/parquet::parquet-struct-object)
6.79- (,@(mapcar (lambda (f)
6.80- (let ((fdoc (parquet-struct-field-doc f))
6.81- (fname (snakecase-name-to-lisp-name
6.82- (parquet-struct-field-name f))))
6.83- `(,(intern fname :dat/parquet)
6.84- ,@(when fdoc `(:documentation ,fdoc))
6.85- :initarg ,(keywordicate fname)
6.86- ;; TODO 2024-07-12:
6.87- ,@(when (equal "optional" (parquet-struct-field-required f))
6.88- `(:initform nil))
6.89- ,@(when-let ((ty (convert-parquet-struct-field-type f)))
6.90- `(:type ,ty)))))
6.91- fields))
6.92- ,@(when doc `((:documentation ,doc))))
6.93- ',class-name)))))
6.94-
6.95-(defmacro define-parquet-structs ()
6.96- `(list
6.97- ,@(%define-parquet-structs)))
6.98+ (defclass ,class-name (dat/parquet::parquet-object)
6.99+ (,@(mapcar (lambda (f)
6.100+ (let ((fdoc (parquet-struct-field-doc f))
6.101+ (fname (snakecase-name-to-lisp-name
6.102+ (parquet-struct-field-name f))))
6.103+ `(,(intern fname :dat/parquet)
6.104+ ,@(when fdoc `(:documentation ,fdoc))
6.105+ :initarg ,(keywordicate fname)
6.106+ ;; TODO 2024-07-12:
6.107+ ,@(when (equal "optional" (parquet-struct-field-required f))
6.108+ `(:initform nil))
6.109+ ,@(when-let ((ty (convert-parquet-struct-field-type f)))
6.110+ `(:type ,ty)))))
6.111+ fields))
6.112+ ,@(when doc `((:documentation ,doc)))))))))
6.113
6.114 (defmacro define-parquet-type (name opts &body body)
6.115 "Define a parquet type with DEFTYPE which maps to LISP-TYPE."
6.116 `(deftype ,(intern (concatenate 'string "PARQUET-" (substitute #\- #\_ name)) :dat/parquet) ,opts ,@body))
6.117
6.118-(defun define-parquet-types ()
6.119- "Define all known values in *PARQUET-TYPES* using DEFINE-PARQUET-TYPE (DEFTYPE)."
6.120- (list
6.121- (define-parquet-type "BOOLEAN" () 'boolean)
6.122- (define-parquet-type "INT32" () '(signed-byte 32))
6.123- (define-parquet-type "INT64" () '(signed-byte 64))
6.124- (define-parquet-type "INT96" () '(signed-byte 96))
6.125- (define-parquet-type "FLOAT" () 'float)
6.126- (define-parquet-type "DOUBLE" () 'double-float)
6.127- (define-parquet-type "BYTE_ARRAY" (&optional size) `(octet-vector ,size))
6.128- (define-parquet-type "FIXED_LEN_BYTE_ARRAY" (size) `(octet-vector ,size))))
6.129+(defun parse-parquet-thrift-definitions (&key (input *parquet-json-file*)
6.130+ (output #.(asdf:system-relative-pathname :dat "parquet/thrift.lisp")))
6.131+ (init-parquet-json input)
6.132+ (with-open-file (defs output :direction :output :if-exists :supersede :if-does-not-exist :create)
6.133+ (format defs ";;; ~a --- Parquet Thrift Definitions -*- buffer-read-only:t -*-
6.134+
6.135+;; input = ~a
6.136+
6.137+;; This file was generated automatically by
6.138+;; DAT/PARQUET/GEN:PARSE-PARQUET-THRIFT-DEFINITIONS
6.139+
6.140+;; Do not modify.
6.141
6.142-(defun load-parquet (&key (file *parquet-json-file*))
6.143- (init-parquet-json file)
6.144- (with-package (:dat/parquet)
6.145- (let ((types (define-parquet-types)))
6.146- (export types))
6.147- (export (define-parquet-structs))
6.148- (export *parquet-enums*)))
6.149+;;; Code:
6.150+(in-package :dat/parquet)" output input)
6.151+ (format defs "~2%")
6.152+ (let ((enums '((define-parquet-enum types "Type")
6.153+ (define-parquet-enum converted-types "ConvertedType")
6.154+ (define-parquet-enum field-repetition-types "FieldRepetitionType")
6.155+ (define-parquet-enum encodings "Encoding")
6.156+ (define-parquet-enum compression-codecs "CompressionCodec")
6.157+ (define-parquet-enum page-types "PageType")
6.158+ (define-parquet-enum boundary-orders "BoundaryOrder")))
6.159+ (types '((define-parquet-type "BOOLEAN" () 'boolean)
6.160+ (define-parquet-type "INT32" () '(signed-byte 32))
6.161+ (define-parquet-type "INT64" () '(signed-byte 64))
6.162+ (define-parquet-type "INT96" () '(signed-byte 96))
6.163+ (define-parquet-type "FLOAT" () 'float)
6.164+ (define-parquet-type "DOUBLE" () 'double-float)
6.165+ (define-parquet-type "BYTE_ARRAY" (&optional size) `(octet-vector ,size))
6.166+ (define-parquet-type "FIXED_LEN_BYTE_ARRAY" (size) `(octet-vector ,size))))
6.167+ (structs (mapcar #'macroexpand-1 (%define-parquet-structs))))
6.168+ ;; expands to a progn, so we just take the cdr
6.169+ (dolist (en enums)
6.170+ (dolist (f (cdr (macroexpand en)))
6.171+ (write f :stream defs :case :downcase :readably t)
6.172+ (terpri defs)))
6.173+ (dolist (ty types)
6.174+ (dolist (f (cdr (macroexpand ty)))
6.175+ (write f :stream defs :case :downcase :readably t)
6.176+ (terpri defs)))
6.177+ (dolist (st structs)
6.178+ (dolist (f (cdr (macroexpand st)))
6.179+ (write f :stream defs :case :downcase :readably t)
6.180+ (terpri defs))))))
7.1--- a/lisp/lib/dat/parquet/obj.lisp Sun Sep 01 21:10:08 2024 -0400
7.2+++ b/lisp/lib/dat/parquet/obj.lisp Mon Sep 02 18:31:19 2024 -0400
7.3@@ -8,21 +8,3 @@
7.4 (defclass parquet-object () ())
7.5
7.6 (defclass parquet-struct-object (parquet-object) ())
7.7-
7.8-(eval-always
7.9- (dat/parquet/gen::load-parquet))
7.10-
7.11-(macrolet ((def-parquet-type (name)
7.12- (let ((var-name (symbolicate "*" name "S*"))
7.13- (name1 (symbolicate name "*")))
7.14- `(progn
7.15- (deftype ,name () `(member ,,var-name))
7.16- (defun ,name (d) (position d ,var-name :test 'eql))
7.17- (defun ,name1 (n) (elt ,var-name n))))))
7.18- (def-parquet-type parquet-compression-codec)
7.19- (def-parquet-type parquet-boundary-order)
7.20- (def-parquet-type parquet-encoding)
7.21- (def-parquet-type parquet-field-repetition-type)
7.22- (def-parquet-type parquet-type)
7.23- (def-parquet-type parquet-converted-type)
7.24- (def-parquet-type parquet-page-type))
8.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
8.2+++ b/lisp/lib/dat/parquet/thrift.lisp Mon Sep 02 18:31:19 2024 -0400
8.3@@ -0,0 +1,1150 @@
8.4+;;; /home/ellis/comp/core/lisp/lib/dat/parquet/thrift.lisp --- Parquet Thrift Definitions -*- buffer-read-only:t -*-
8.5+
8.6+;; input = /home/ellis/comp/core/.stash/parquet.json
8.7+
8.8+;; This file was generated automatically by
8.9+;; DAT/PARQUET/GEN:PARSE-PARQUET-THRIFT-DEFINITIONS
8.10+
8.11+;; Do not modify.
8.12+
8.13+;;; Code:
8.14+(in-package :dat/parquet)
8.15+
8.16+(defun parquet-json-types ()
8.17+ (mapcar
8.18+ (lambda (dat/parquet/gen::x)
8.19+ (sb-int:keywordicate
8.20+ (dat/parquet/gen::snakecase-name-to-lisp-name
8.21+ (dat/json:json-getf dat/parquet/gen::x "name"))))
8.22+ (dat/parquet/gen::parquet-json-enum-getf "Type")))
8.23+(defparameter dat/parquet:*parquet-types* (parquet-json-types))
8.24+(defun parquet-json-converted-types ()
8.25+ (mapcar
8.26+ (lambda (dat/parquet/gen::x)
8.27+ (sb-int:keywordicate
8.28+ (dat/parquet/gen::snakecase-name-to-lisp-name
8.29+ (dat/json:json-getf dat/parquet/gen::x "name"))))
8.30+ (dat/parquet/gen::parquet-json-enum-getf "ConvertedType")))
8.31+(defparameter dat/parquet:*parquet-converted-types*
8.32+ (parquet-json-converted-types))
8.33+(defun parquet-json-field-repetition-types ()
8.34+ (mapcar
8.35+ (lambda (dat/parquet/gen::x)
8.36+ (sb-int:keywordicate
8.37+ (dat/parquet/gen::snakecase-name-to-lisp-name
8.38+ (dat/json:json-getf dat/parquet/gen::x "name"))))
8.39+ (dat/parquet/gen::parquet-json-enum-getf "FieldRepetitionType")))
8.40+(defparameter dat/parquet:*parquet-field-repetition-types*
8.41+ (parquet-json-field-repetition-types))
8.42+(defun parquet-json-encodings ()
8.43+ (mapcar
8.44+ (lambda (dat/parquet/gen::x)
8.45+ (sb-int:keywordicate
8.46+ (dat/parquet/gen::snakecase-name-to-lisp-name
8.47+ (dat/json:json-getf dat/parquet/gen::x "name"))))
8.48+ (dat/parquet/gen::parquet-json-enum-getf "Encoding")))
8.49+(defparameter dat/parquet:*parquet-encodings* (parquet-json-encodings))
8.50+(defun parquet-json-compression-codecs ()
8.51+ (mapcar
8.52+ (lambda (dat/parquet/gen::x)
8.53+ (sb-int:keywordicate
8.54+ (dat/parquet/gen::snakecase-name-to-lisp-name
8.55+ (dat/json:json-getf dat/parquet/gen::x "name"))))
8.56+ (dat/parquet/gen::parquet-json-enum-getf "CompressionCodec")))
8.57+(defparameter dat/parquet:*parquet-compression-codecs*
8.58+ (parquet-json-compression-codecs))
8.59+(defun parquet-json-page-types ()
8.60+ (mapcar
8.61+ (lambda (dat/parquet/gen::x)
8.62+ (sb-int:keywordicate
8.63+ (dat/parquet/gen::snakecase-name-to-lisp-name
8.64+ (dat/json:json-getf dat/parquet/gen::x "name"))))
8.65+ (dat/parquet/gen::parquet-json-enum-getf "PageType")))
8.66+(defparameter dat/parquet:*parquet-page-types* (parquet-json-page-types))
8.67+(defun parquet-json-boundary-orders ()
8.68+ (mapcar
8.69+ (lambda (dat/parquet/gen::x)
8.70+ (sb-int:keywordicate
8.71+ (dat/parquet/gen::snakecase-name-to-lisp-name
8.72+ (dat/json:json-getf dat/parquet/gen::x "name"))))
8.73+ (dat/parquet/gen::parquet-json-enum-getf "BoundaryOrder")))
8.74+(defparameter dat/parquet:*parquet-boundary-orders*
8.75+ (parquet-json-boundary-orders))
8.76+(eval-when (:compile-toplevel :load-toplevel :execute)
8.77+ (sb-impl::%deftype 'dat/parquet:parquet-boolean
8.78+ (sb-impl::constant-type-expander
8.79+ 'dat/parquet:parquet-boolean (progn 'boolean))
8.80+ (sb-c:source-location)))
8.81+(eval-when (:compile-toplevel :load-toplevel :execute)
8.82+ (sb-impl::%deftype 'dat/parquet:parquet-int32
8.83+ (sb-impl::constant-type-expander
8.84+ 'dat/parquet:parquet-int32 (progn '(signed-byte 32)))
8.85+ (sb-c:source-location)))
8.86+(eval-when (:compile-toplevel :load-toplevel :execute)
8.87+ (sb-impl::%deftype 'dat/parquet:parquet-int64
8.88+ (sb-impl::constant-type-expander
8.89+ 'dat/parquet:parquet-int64 (progn '(signed-byte 64)))
8.90+ (sb-c:source-location)))
8.91+(eval-when (:compile-toplevel :load-toplevel :execute)
8.92+ (sb-impl::%deftype 'dat/parquet:parquet-int96
8.93+ (sb-impl::constant-type-expander
8.94+ 'dat/parquet:parquet-int96 (progn '(signed-byte 96)))
8.95+ (sb-c:source-location)))
8.96+(eval-when (:compile-toplevel :load-toplevel :execute)
8.97+ (sb-impl::%deftype 'dat/parquet:parquet-float
8.98+ (sb-impl::constant-type-expander
8.99+ 'dat/parquet:parquet-float (progn 'float))
8.100+ (sb-c:source-location)))
8.101+(eval-when (:compile-toplevel :load-toplevel :execute)
8.102+ (sb-impl::%deftype 'dat/parquet:parquet-double
8.103+ (sb-impl::constant-type-expander
8.104+ 'dat/parquet:parquet-double (progn 'double-float))
8.105+ (sb-c:source-location)))
8.106+(eval-when (:compile-toplevel :load-toplevel :execute)
8.107+ (sb-impl::%deftype 'dat/parquet:parquet-byte-array
8.108+ (sb-int:named-lambda (sb-impl::type-expander
8.109+ dat/parquet:parquet-byte-array)
8.110+ (#:expr)
8.111+ (declare
8.112+ (sb-c::lambda-list (&optional dat/parquet/gen::size)))
8.113+ (sb-int:named-ds-bind (:macro
8.114+ dat/parquet:parquet-byte-array
8.115+ . deftype)
8.116+ (&optional dat/parquet/gen::size)
8.117+ (cdr #:expr)
8.118+ (declare (sb-c::constant-value dat/parquet/gen::size))
8.119+ (block dat/parquet:parquet-byte-array
8.120+ `(std/type:octet-vector ,dat/parquet/gen::size))))
8.121+ nil))
8.122+(eval-when (:compile-toplevel :load-toplevel :execute)
8.123+ (sb-impl::%deftype 'dat/parquet:parquet-fixed-len-byte-array
8.124+ (sb-int:named-lambda (sb-impl::type-expander
8.125+ dat/parquet:parquet-fixed-len-byte-array)
8.126+ (#:expr)
8.127+ (declare (sb-c::lambda-list (dat/parquet/gen::size)))
8.128+ (sb-int:named-ds-bind (:macro
8.129+ dat/parquet:parquet-fixed-len-byte-array
8.130+ . deftype)
8.131+ (dat/parquet/gen::size)
8.132+ (cdr #:expr)
8.133+ (declare (sb-c::constant-value dat/parquet/gen::size))
8.134+ (block dat/parquet:parquet-fixed-len-byte-array
8.135+ `(std/type:octet-vector ,dat/parquet/gen::size))))
8.136+ nil))
8.137+(defclass dat/parquet:parquet-size-statistics (dat/parquet:parquet-object)
8.138+ ((dat/parquet::unencoded-byte-array-data-bytes :documentation
8.139+ "The number of physical bytes stored for BYTE_ARRAY data values assuming
8.140+no encoding. This is exclusive of the bytes needed to store the length of
8.141+each byte array. In other words, this field is equivalent to the `(size
8.142+of PLAIN-ENCODING the byte array values) - (4 bytes * number of values
8.143+written)`. To determine unencoded sizes of other types readers can use
8.144+schema information multiplied by the number of non-null and null values.
8.145+The number of null\\non-null values can be inferred from the histograms
8.146+below.
8.147+
8.148+For example, if a column chunk is dictionary-encoded with dictionary
8.149+[\\a\\, \\bc\\, \\cde\\], and a data page contains the indices [0, 0, 1, 2],
8.150+then this value for that data page should be 7 (1 + 1 + 2 + 3).
8.151+
8.152+This field should only be set for types that use BYTE_ARRAY as their
8.153+physical type.
8.154+"
8.155+ :initarg :unencoded-byte-array-data-bytes :initform nil :type
8.156+ (or null (signed-byte 64)))
8.157+ (dat/parquet::repetition-level-histogram :documentation
8.158+ "When present, there is expected to be one element corresponding to each
8.159+repetition (i.e. size=max repetition_level+1) where each element
8.160+represents the number of times the repetition level was observed in the
8.161+data.
8.162+
8.163+This field may be omitted if max_repetition_level is 0 without loss
8.164+of information.
8.165+
8.166+"
8.167+ :initarg :repetition-level-histogram :initform nil :type
8.168+ (or null (vector (signed-byte 64))))
8.169+ (dat/parquet::definition-level-histogram :documentation
8.170+ "Same as repetition_level_histogram except for definition levels.
8.171+
8.172+This field may be omitted if max_definition_level is 0 or 1 without
8.173+loss of information.
8.174+
8.175+"
8.176+ :initarg :definition-level-histogram :initform nil :type
8.177+ (or null (vector (signed-byte 64)))))
8.178+ (:documentation
8.179+ "A structure for capturing metadata for estimating the unencoded,
8.180+uncompressed size of data written. This is useful for readers to estimate
8.181+how much memory is needed to reconstruct data in their memory model and for
8.182+fine grained filter pushdown on nested structures (the histograms contained
8.183+in this structure can help determine the number of nulls at a particular
8.184+nesting level and maximum length of lists).
8.185+"))
8.186+(defclass dat/parquet:parquet-statistics (dat/parquet:parquet-object)
8.187+ ((max :documentation
8.188+ "DEPRECATED: min and max value of the column. Use min_value and max_value.
8.189+
8.190+Values are encoded using PLAIN encoding, except that variable-length byte
8.191+arrays do not include a length prefix.
8.192+
8.193+These fields encode min and max values determined by signed comparison
8.194+only. New files should use the correct order for a column's logical type
8.195+and store the values in the min_value and max_value fields.
8.196+
8.197+To support older readers, these may be set when the column order is
8.198+signed.
8.199+"
8.200+ :initarg :max :initform nil :type
8.201+ (or null std/type:octet-vector))
8.202+ (min :initarg :min :initform nil :type
8.203+ (or null std/type:octet-vector))
8.204+ (dat/parquet::null-count :documentation
8.205+ "count of null value in the column
8.206+"
8.207+ :initarg :null-count :initform nil :type
8.208+ (or null (signed-byte 64)))
8.209+ (dat/parquet::distinct-count :documentation
8.210+ "count of distinct values occurring
8.211+"
8.212+ :initarg :distinct-count :initform nil :type
8.213+ (or null (signed-byte 64)))
8.214+ (dat/parquet::max-value :documentation
8.215+ "Lower and upper bound values for the column, determined by its ColumnOrder.
8.216+
8.217+These may be the actual minimum and maximum values found on a page or column
8.218+chunk, but can also be (more compact) values that do not exist on a page or
8.219+column chunk. For example, instead of storing \\Blart Versenwald III\\, a writer
8.220+may set min_value=\\B\\, max_value=\\C\\. Such more compact values must still be
8.221+valid values within the column's logical type.
8.222+
8.223+Values are encoded using PLAIN encoding, except that variable-length byte
8.224+arrays do not include a length prefix.
8.225+"
8.226+ :initarg :max-value :initform nil :type
8.227+ (or null std/type:octet-vector))
8.228+ (dat/parquet::min-value :initarg :min-value :initform nil :type
8.229+ (or null std/type:octet-vector))
8.230+ (dat/parquet::is-max-value-exact :documentation
8.231+ "If true, max_value is the actual maximum value for a column
8.232+"
8.233+ :initarg :is-max-value-exact :initform nil :type (or null boolean))
8.234+ (dat/parquet::is-min-value-exact :documentation
8.235+ "If true, min_value is the actual minimum value for a column
8.236+"
8.237+ :initarg :is-min-value-exact :initform nil :type
8.238+ (or null boolean)))
8.239+ (:documentation "Statistics per row group and per page
8.240+All fields are optional.
8.241+"))
8.242+(defclass dat/parquet:parquet-string-type (dat/parquet:parquet-object) nil
8.243+ (:documentation "Empty structs to use as logical type annotations
8.244+"))
8.245+(defclass dat/parquet:parquet-uuid-type (dat/parquet:parquet-object) nil)
8.246+(defclass dat/parquet:parquet-map-type (dat/parquet:parquet-object) nil)
8.247+(defclass dat/parquet:parquet-list-type (dat/parquet:parquet-object) nil)
8.248+(defclass dat/parquet:parquet-enum-type (dat/parquet:parquet-object) nil)
8.249+(defclass dat/parquet:parquet-date-type (dat/parquet:parquet-object) nil)
8.250+(defclass dat/parquet:parquet-float16-type (dat/parquet:parquet-object) nil)
8.251+(defclass dat/parquet:parquet-null-type (dat/parquet:parquet-object) nil
8.252+ (:documentation
8.253+ "Logical type to annotate a column that is always null.
8.254+
8.255+Sometimes when discovering the schema of existing data, values are always
8.256+null and the physical type can't be determined. This annotation signals
8.257+the case where the physical type was guessed from all null values.
8.258+"))
8.259+(defclass dat/parquet:parquet-decimal-type (dat/parquet:parquet-object)
8.260+ ((dat/parquet::scale :initarg :scale :type (signed-byte 32))
8.261+ (dat/parquet::precision :initarg :precision :type (signed-byte 32)))
8.262+ (:documentation "Decimal logical type annotation
8.263+
8.264+Scale must be zero or a positive integer less than or equal to the precision.
8.265+Precision must be a non-zero positive integer.
8.266+
8.267+To maintain forward-compatibility in v1, implementations using this logical
8.268+type must also set scale and precision on the annotated SchemaElement.
8.269+
8.270+Allowed for physical types: INT32, INT64, FIXED_LEN_BYTE_ARRAY, and BYTE_ARRAY.
8.271+"))
8.272+(defclass dat/parquet:parquet-milli-seconds (dat/parquet:parquet-object) nil
8.273+ (:documentation "Time units for logical types
8.274+"))
8.275+(defclass dat/parquet:parquet-micro-seconds (dat/parquet:parquet-object) nil)
8.276+(defclass dat/parquet:parquet-nano-seconds (dat/parquet:parquet-object) nil)
8.277+(defclass dat/parquet:parquet-time-unit (dat/parquet:parquet-object)
8.278+ ((dat/parquet::millis :initarg :millis :initform nil :type
8.279+ (or null dat/parquet:parquet-milli-seconds))
8.280+ (dat/parquet::micros :initarg :micros :initform nil :type
8.281+ (or null dat/parquet:parquet-micro-seconds))
8.282+ (dat/parquet::nanos :initarg :nanos :initform nil :type
8.283+ (or null dat/parquet:parquet-nano-seconds))))
8.284+(defclass dat/parquet:parquet-timestamp-type (dat/parquet:parquet-object)
8.285+ ((dat/parquet::isadjustedtoutc :initarg :isadjustedtoutc :type
8.286+ boolean)
8.287+ (dat/parquet::unit :initarg :unit :type
8.288+ dat/parquet:parquet-time-unit))
8.289+ (:documentation "Timestamp logical type annotation
8.290+
8.291+Allowed for physical types: INT64
8.292+"))
8.293+(defclass dat/parquet:parquet-time-type (dat/parquet:parquet-object)
8.294+ ((dat/parquet::isadjustedtoutc :initarg :isadjustedtoutc :type
8.295+ boolean)
8.296+ (dat/parquet::unit :initarg :unit :type
8.297+ dat/parquet:parquet-time-unit))
8.298+ (:documentation "Time logical type annotation
8.299+
8.300+Allowed for physical types: INT32 (millis), INT64 (micros, nanos)
8.301+"))
8.302+(defclass dat/parquet:parquet-int-type (dat/parquet:parquet-object)
8.303+ ((dat/parquet::bitwidth :initarg :bitwidth)
8.304+ (dat/parquet::issigned :initarg :issigned :type boolean))
8.305+ (:documentation "Integer logical type annotation
8.306+
8.307+bitWidth must be 8, 16, 32, or 64.
8.308+
8.309+Allowed for physical types: INT32, INT64
8.310+"))
8.311+(defclass dat/parquet:parquet-json-type (dat/parquet:parquet-object) nil
8.312+ (:documentation "Embedded JSON logical type annotation
8.313+
8.314+Allowed for physical types: BYTE_ARRAY
8.315+"))
8.316+(defclass dat/parquet:parquet-bson-type (dat/parquet:parquet-object) nil
8.317+ (:documentation "Embedded BSON logical type annotation
8.318+
8.319+Allowed for physical types: BYTE_ARRAY
8.320+"))
8.321+(defclass dat/parquet:parquet-logical-type (dat/parquet:parquet-object)
8.322+ ((string :initarg :string :initform nil :type
8.323+ (or null dat/parquet:parquet-string-type))
8.324+ (map :initarg :map :initform nil :type
8.325+ (or null dat/parquet:parquet-map-type))
8.326+ (list :initarg :list :initform nil :type
8.327+ (or null dat/parquet:parquet-list-type))
8.328+ (dat/parquet::enum :initarg :enum :initform nil :type
8.329+ (or null dat/parquet:parquet-enum-type))
8.330+ (dat/parquet::decimal :initarg :decimal :initform nil :type
8.331+ (or null dat/parquet:parquet-decimal-type))
8.332+ (dat/parquet::date :initarg :date :initform nil :type
8.333+ (or null dat/parquet:parquet-date-type))
8.334+ (time
8.335+ :initarg
8.336+ :time
8.337+ :initform
8.338+ nil
8.339+ :type
8.340+ (or null dat/parquet:parquet-time-type))
8.341+ (dat/parquet::timestamp :initarg :timestamp :initform nil :type
8.342+ (or null dat/parquet:parquet-timestamp-type))
8.343+ (integer :initarg :integer :initform nil :type
8.344+ (or null dat/parquet:parquet-int-type))
8.345+ (dat/parquet::unknown :initarg :unknown :initform nil :type
8.346+ (or null dat/parquet:parquet-null-type))
8.347+ (dat/parquet::json :initarg :json :initform nil :type
8.348+ (or null dat/parquet:parquet-json-type))
8.349+ (dat/parquet::bson :initarg :bson :initform nil :type
8.350+ (or null dat/parquet:parquet-bson-type))
8.351+ (dat/parquet::uuid :initarg :uuid :initform nil :type
8.352+ (or null dat/parquet:parquet-uuid-type))
8.353+ (dat/parquet::float16 :initarg :float16 :initform nil :type
8.354+ (or null dat/parquet:parquet-float16-type)))
8.355+ (:documentation "LogicalType annotations to replace ConvertedType.
8.356+
8.357+To maintain compatibility, implementations using LogicalType for a
8.358+SchemaElement must also set the corresponding ConvertedType (if any)
8.359+from the following table.
8.360+"))
8.361+(defclass dat/parquet:parquet-schema-element (dat/parquet:parquet-object)
8.362+ ((type :documentation
8.363+ "Data type for this field. Not set if the current element is a non-leaf node
8.364+"
8.365+ :initarg :type :initform nil :type
8.366+ (or null dat/parquet::parquet-type))
8.367+ (dat/parquet::type-length :documentation
8.368+ "If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values.
8.369+Otherwise, if specified, this is the maximum bit length to store any of the values.
8.370+(e.g. a low cardinality INT col could have this set to 3). Note that this is
8.371+in the schema, and therefore fixed for the entire file.
8.372+"
8.373+ :initarg :type-length :initform nil :type
8.374+ (or null (signed-byte 32)))
8.375+ (dat/parquet::repetition-type :documentation
8.376+ "repetition of the field. The root of the schema does not have a repetition_type.
8.377+All other nodes must have one
8.378+"
8.379+ :initarg :repetition-type :initform nil :type
8.380+ (or null dat/parquet::parquet-field-repetition-type))
8.381+ (dat/parquet::name :documentation "Name of the field in the schema
8.382+"
8.383+ :initarg :name :type string)
8.384+ (dat/parquet::num-children :documentation
8.385+ "Nested fields. Since thrift does not support nested fields,
8.386+the nesting is flattened to a single list by a depth-first traversal.
8.387+The children count is used to construct the nested relationship.
8.388+This field is not set when the element is a primitive type
8.389+"
8.390+ :initarg :num-children :initform nil :type
8.391+ (or null (signed-byte 32)))
8.392+ (dat/parquet::converted-type :documentation
8.393+ "DEPRECATED: When the schema is the result of a conversion from another model.
8.394+Used to record the original type to help with cross conversion.
8.395+
8.396+This is superseded by logicalType.
8.397+"
8.398+ :initarg :converted-type :initform nil :type
8.399+ (or null dat/parquet::parquet-converted-type))
8.400+ (dat/parquet::scale :documentation
8.401+ "DEPRECATED: Used when this column contains decimal data.
8.402+See the DECIMAL converted type for more details.
8.403+
8.404+This is superseded by using the DecimalType annotation in logicalType.
8.405+"
8.406+ :initarg :scale :initform nil :type (or null (signed-byte 32)))
8.407+ (dat/parquet::precision :initarg :precision :initform nil :type
8.408+ (or null (signed-byte 32)))
8.409+ (dat/parquet::field-id :documentation
8.410+ "When the original schema supports field ids, this will save the
8.411+original field id in the parquet schema
8.412+"
8.413+ :initarg :field-id :initform nil :type (or null (signed-byte 32)))
8.414+ (dat/parquet::logicaltype :documentation
8.415+ "The logical type of this SchemaElement
8.416+
8.417+LogicalType replaces ConvertedType, but ConvertedType is still required
8.418+for some logical types to ensure forward-compatibility in format v1.
8.419+"
8.420+ :initarg :logicaltype :initform nil :type
8.421+ (or null dat/parquet:parquet-logical-type)))
8.422+ (:documentation "Represents a element inside a schema definition.
8.423+ - if it is a group (inner node) then type is undefined and num_children is defined
8.424+ - if it is a primitive type (leaf) then type is defined and num_children is undefined
8.425+the nodes are listed in depth first traversal order.
8.426+"))
8.427+(defclass dat/parquet:parquet-data-page-header (dat/parquet:parquet-object)
8.428+ ((dat/parquet::num-values :documentation
8.429+ "Number of values, including NULLs, in this data page.
8.430+
8.431+If a OffsetIndex is present, a page must begin at a row
8.432+boundary (repetition_level = 0). Otherwise, pages may begin
8.433+within a row (repetition_level > 0).
8.434+
8.435+"
8.436+ :initarg :num-values :type (signed-byte 32))
8.437+ (dat/parquet::encoding :documentation
8.438+ "Encoding used for this data page *
8.439+"
8.440+ :initarg :encoding :type dat/parquet::parquet-encoding)
8.441+ (dat/parquet::definition-level-encoding :documentation
8.442+ "Encoding used for definition levels *
8.443+"
8.444+ :initarg :definition-level-encoding :type
8.445+ dat/parquet::parquet-encoding)
8.446+ (dat/parquet::repetition-level-encoding :documentation
8.447+ "Encoding used for repetition levels *
8.448+"
8.449+ :initarg :repetition-level-encoding :type
8.450+ dat/parquet::parquet-encoding)
8.451+ (dat/parquet::statistics :documentation
8.452+ "Optional statistics for the data in this page *
8.453+"
8.454+ :initarg :statistics :initform nil :type
8.455+ (or null dat/parquet:parquet-statistics)))
8.456+ (:documentation "Data page header
8.457+"))
8.458+(defclass dat/parquet:parquet-index-page-header (dat/parquet:parquet-object)
8.459+ nil)
8.460+(defclass dat/parquet:parquet-dictionary-page-header
8.461+ (dat/parquet:parquet-object)
8.462+ ((dat/parquet::num-values :documentation
8.463+ "Number of values in the dictionary *
8.464+"
8.465+ :initarg :num-values :type (signed-byte 32))
8.466+ (dat/parquet::encoding :documentation
8.467+ "Encoding using this dictionary page *
8.468+"
8.469+ :initarg :encoding :type dat/parquet::parquet-encoding)
8.470+ (dat/parquet::is-sorted :documentation
8.471+ "If true, the entries in the dictionary are sorted in ascending order *
8.472+"
8.473+ :initarg :is-sorted :initform nil :type (or null boolean)))
8.474+ (:documentation
8.475+ "The dictionary page must be placed at the first position of the column chunk
8.476+if it is partly or completely dictionary encoded. At most one dictionary page
8.477+can be placed in a column chunk.
8.478+
8.479+"))
8.480+(defclass dat/parquet:parquet-data-page-header-v2 (dat/parquet:parquet-object)
8.481+ ((dat/parquet::num-values :documentation
8.482+ "Number of values, including NULLs, in this data page. *
8.483+"
8.484+ :initarg :num-values :type (signed-byte 32))
8.485+ (dat/parquet::num-nulls :documentation
8.486+ "Number of NULL values, in this data page.
8.487+Number of non-null = num_values - num_nulls which is also the number of values in the data section *
8.488+"
8.489+ :initarg :num-nulls :type (signed-byte 32))
8.490+ (dat/parquet::num-rows :documentation
8.491+ "Number of rows in this data page. Every page must begin at a
8.492+row boundary (repetition_level = 0): rows must **not** be
8.493+split across page boundaries when using V2 data pages.
8.494+
8.495+"
8.496+ :initarg :num-rows :type (signed-byte 32))
8.497+ (dat/parquet::encoding :documentation
8.498+ "Encoding used for data in this page *
8.499+"
8.500+ :initarg :encoding :type dat/parquet::parquet-encoding)
8.501+ (dat/parquet::definition-levels-byte-length :documentation
8.502+ "Length of the definition levels
8.503+"
8.504+ :initarg :definition-levels-byte-length :type (signed-byte 32))
8.505+ (dat/parquet::repetition-levels-byte-length :documentation
8.506+ "Length of the repetition levels
8.507+"
8.508+ :initarg :repetition-levels-byte-length :type (signed-byte 32))
8.509+ (dat/parquet::is-compressed :documentation
8.510+ "Whether the values are compressed.
8.511+Which means the section of the page between
8.512+definition_levels_byte_length + repetition_levels_byte_length + 1 and compressed_page_size (included)
8.513+is compressed with the compression_codec.
8.514+If missing it is considered compressed
8.515+"
8.516+ :initarg :is-compressed :initform nil :type (or null boolean))
8.517+ (dat/parquet::statistics :documentation
8.518+ "Optional statistics for the data in this page *
8.519+"
8.520+ :initarg :statistics :initform nil :type
8.521+ (or null dat/parquet:parquet-statistics)))
8.522+ (:documentation
8.523+ "New page format allowing reading levels without decompressing the data
8.524+Repetition and definition levels are uncompressed
8.525+The remaining section containing the data is compressed if is_compressed is true
8.526+
8.527+"))
8.528+(defclass dat/parquet:parquet-split-block-algorithm
8.529+ (dat/parquet:parquet-object) nil
8.530+ (:documentation "Block-based algorithm type annotation. *
8.531+"))
8.532+(defclass dat/parquet:parquet-bloom-filter-algorithm
8.533+ (dat/parquet:parquet-object)
8.534+ ((block :documentation
8.535+ "Block-based Bloom filter. *
8.536+"
8.537+ :initarg
8.538+ :block
8.539+ :initform
8.540+ nil
8.541+ :type
8.542+ (or null dat/parquet:parquet-split-block-algorithm)))
8.543+ (:documentation "The algorithm used in Bloom filter. *
8.544+"))
8.545+(defclass dat/parquet:parquet-xx-hash (dat/parquet:parquet-object) nil
8.546+ (:documentation
8.547+ "Hash strategy type annotation. xxHash is an extremely fast non-cryptographic hash
8.548+algorithm. It uses 64 bits version of xxHash.
8.549+
8.550+"))
8.551+(defclass dat/parquet:parquet-bloom-filter-hash (dat/parquet:parquet-object)
8.552+ ((dat/parquet::xxhash :documentation "xxHash Strategy. *
8.553+"
8.554+ :initarg :xxhash :initform nil :type
8.555+ (or null dat/parquet:parquet-xx-hash)))
8.556+ (:documentation
8.557+ "The hash function used in Bloom filter. This function takes the hash of a column value
8.558+using plain encoding.
8.559+
8.560+"))
8.561+(defclass dat/parquet:parquet-uncompressed (dat/parquet:parquet-object) nil
8.562+ (:documentation "The compression used in the Bloom filter.
8.563+
8.564+"))
8.565+(defclass dat/parquet:parquet-bloom-filter-compression
8.566+ (dat/parquet:parquet-object)
8.567+ ((dat/parquet::uncompressed :initarg :uncompressed :initform nil
8.568+ :type (or null dat/parquet:parquet-uncompressed))))
8.569+(defclass dat/parquet:parquet-bloom-filter-header (dat/parquet:parquet-object)
8.570+ ((dat/parquet::numbytes :documentation "The size of bitset in bytes *
8.571+"
8.572+ :initarg :numbytes :type (signed-byte 32))
8.573+ (dat/parquet::algorithm :documentation
8.574+ "The algorithm for setting bits. *
8.575+"
8.576+ :initarg :algorithm :type
8.577+ dat/parquet:parquet-bloom-filter-algorithm)
8.578+ (dat/parquet::hash :documentation
8.579+ "The hash function used for Bloom filter. *
8.580+"
8.581+ :initarg :hash :type dat/parquet:parquet-bloom-filter-hash)
8.582+ (dat/parquet::compression :documentation
8.583+ "The compression used in the Bloom filter *
8.584+"
8.585+ :initarg :compression :type
8.586+ dat/parquet:parquet-bloom-filter-compression))
8.587+ (:documentation
8.588+ "Bloom filter header is stored at beginning of Bloom filter data of each column
8.589+and followed by its bitset.
8.590+
8.591+"))
8.592+(defclass dat/parquet:parquet-page-header (dat/parquet:parquet-object)
8.593+ ((type :documentation
8.594+ "the type of the page: indicates which of the *_header fields is set *
8.595+"
8.596+ :initarg :type :type dat/parquet::parquet-page-type)
8.597+ (dat/parquet::uncompressed-page-size :documentation
8.598+ "Uncompressed page size in bytes (not including this header) *
8.599+"
8.600+ :initarg :uncompressed-page-size :type (signed-byte 32))
8.601+ (dat/parquet::compressed-page-size :documentation
8.602+ "Compressed (and potentially encrypted) page size in bytes, not including this header *
8.603+"
8.604+ :initarg :compressed-page-size :type (signed-byte 32))
8.605+ (dat/parquet::crc :documentation
8.606+ "The 32-bit CRC checksum for the page, to be be calculated as follows:
8.607+
8.608+- The standard CRC32 algorithm is used (with polynomial 0x04C11DB7,
8.609+ the same as in e.g. GZip).
8.610+- All page types can have a CRC (v1 and v2 data pages, dictionary pages,
8.611+ etc.).
8.612+- The CRC is computed on the serialization binary representation of the page
8.613+ (as written to disk), excluding the page header. For example, for v1
8.614+ data pages, the CRC is computed on the concatenation of repetition levels,
8.615+ definition levels and column values (optionally compressed, optionally
8.616+ encrypted).
8.617+- The CRC computation therefore takes place after any compression
8.618+ and encryption steps, if any.
8.619+
8.620+If enabled, this allows for disabling checksumming in HDFS if only a few
8.621+pages need to be read.
8.622+"
8.623+ :initarg :crc :initform nil :type (or null (signed-byte 32)))
8.624+ (dat/parquet::data-page-header :initarg :data-page-header :initform
8.625+ nil :type (or null dat/parquet:parquet-data-page-header))
8.626+ (dat/parquet::index-page-header :initarg :index-page-header
8.627+ :initform nil :type
8.628+ (or null dat/parquet:parquet-index-page-header))
8.629+ (dat/parquet::dictionary-page-header :initarg
8.630+ :dictionary-page-header :initform nil :type
8.631+ (or null dat/parquet:parquet-dictionary-page-header))
8.632+ (dat/parquet::data-page-header-v2 :initarg :data-page-header-v2
8.633+ :initform nil :type
8.634+ (or null dat/parquet:parquet-data-page-header-v2))))
8.635+(defclass dat/parquet:parquet-key-value (dat/parquet:parquet-object)
8.636+ ((dat/parquet::key :initarg :key :type string)
8.637+ (dat/parquet::value :initarg :value :initform nil :type
8.638+ (or null string)))
8.639+ (:documentation "Wrapper struct to store key values
8.640+"))
8.641+(defclass dat/parquet:parquet-sorting-column (dat/parquet:parquet-object)
8.642+ ((dat/parquet::column-idx :documentation
8.643+ "The ordinal position of the column (in this row group) *
8.644+"
8.645+ :initarg :column-idx :type (signed-byte 32))
8.646+ (dat/parquet::descending :documentation
8.647+ "If true, indicates this column is sorted in descending order. *
8.648+"
8.649+ :initarg :descending :type boolean)
8.650+ (dat/parquet::nulls-first :documentation
8.651+ "If true, nulls will come before non-null values, otherwise,
8.652+nulls go at the end.
8.653+"
8.654+ :initarg :nulls-first :type boolean))
8.655+ (:documentation "Sort order within a RowGroup of a leaf column
8.656+"))
8.657+(defclass dat/parquet:parquet-page-encoding-stats (dat/parquet:parquet-object)
8.658+ ((dat/parquet::page-type :documentation
8.659+ "the page type (data\\dic\\...) *
8.660+"
8.661+ :initarg :page-type :type dat/parquet::parquet-page-type)
8.662+ (dat/parquet::encoding :documentation "encoding of the page *
8.663+"
8.664+ :initarg :encoding :type dat/parquet::parquet-encoding)
8.665+ (count :documentation
8.666+ "number of pages of this type with this encoding *
8.667+"
8.668+ :initarg :count :type (signed-byte 32)))
8.669+ (:documentation "statistics of a given page type and encoding
8.670+"))
8.671+(defclass dat/parquet:parquet-column-meta-data (dat/parquet:parquet-object)
8.672+ ((type :documentation "Type of this column *
8.673+"
8.674+ :initarg :type :type dat/parquet::parquet-type)
8.675+ (dat/parquet::encodings :documentation
8.676+ "Set of all encodings used for this column. The purpose is to validate
8.677+whether we can decode those pages. *
8.678+"
8.679+ :initarg :encodings :type (vector dat/parquet::parquet-encoding))
8.680+ (dat/parquet::path-in-schema :documentation "Path in schema *
8.681+"
8.682+ :initarg :path-in-schema :type (vector string))
8.683+ (dat/parquet::codec :documentation "Compression codec *
8.684+"
8.685+ :initarg :codec :type dat/parquet::parquet-compression-codec)
8.686+ (dat/parquet::num-values :documentation
8.687+ "Number of values in this column *
8.688+"
8.689+ :initarg :num-values :type (signed-byte 64))
8.690+ (dat/parquet::total-uncompressed-size :documentation
8.691+ "total byte size of all uncompressed pages in this column chunk (including the headers) *
8.692+"
8.693+ :initarg :total-uncompressed-size :type (signed-byte 64))
8.694+ (dat/parquet::total-compressed-size :documentation
8.695+ "total byte size of all compressed, and potentially encrypted, pages
8.696+in this column chunk (including the headers) *
8.697+"
8.698+ :initarg :total-compressed-size :type (signed-byte 64))
8.699+ (dat/parquet::key-value-metadata :documentation
8.700+ "Optional key\\value metadata *
8.701+"
8.702+ :initarg :key-value-metadata :initform nil :type
8.703+ (or null (vector dat/parquet:parquet-key-value)))
8.704+ (dat/parquet::data-page-offset :documentation
8.705+ "Byte offset from beginning of file to first data page *
8.706+"
8.707+ :initarg :data-page-offset :type (signed-byte 64))
8.708+ (dat/parquet::index-page-offset :documentation
8.709+ "Byte offset from beginning of file to root index page *
8.710+"
8.711+ :initarg :index-page-offset :initform nil :type
8.712+ (or null (signed-byte 64)))
8.713+ (dat/parquet::dictionary-page-offset :documentation
8.714+ "Byte offset from the beginning of file to first (only) dictionary page *
8.715+"
8.716+ :initarg :dictionary-page-offset :initform nil :type
8.717+ (or null (signed-byte 64)))
8.718+ (dat/parquet::statistics :documentation
8.719+ "optional statistics for this column chunk
8.720+"
8.721+ :initarg :statistics :initform nil :type
8.722+ (or null dat/parquet:parquet-statistics))
8.723+ (dat/parquet::encoding-stats :documentation
8.724+ "Set of all encodings used for pages in this column chunk.
8.725+This information can be used to determine if all data pages are
8.726+dictionary encoded for example *
8.727+"
8.728+ :initarg :encoding-stats :initform nil :type
8.729+ (or null (vector dat/parquet:parquet-page-encoding-stats)))
8.730+ (dat/parquet::bloom-filter-offset :documentation
8.731+ "Byte offset from beginning of file to Bloom filter data. *
8.732+"
8.733+ :initarg :bloom-filter-offset :initform nil :type
8.734+ (or null (signed-byte 64)))
8.735+ (dat/parquet::bloom-filter-length :documentation
8.736+ "Size of Bloom filter data including the serialized header, in bytes.
8.737+Added in 2.10 so readers may not read this field from old files and
8.738+it can be obtained after the BloomFilterHeader has been deserialized.
8.739+Writers should write this field so readers can read the bloom filter
8.740+in a single I\\O.
8.741+"
8.742+ :initarg :bloom-filter-length :initform nil :type
8.743+ (or null (signed-byte 32)))
8.744+ (dat/parquet::size-statistics :documentation
8.745+ "Optional statistics to help estimate total memory when converted to in-memory
8.746+representations. The histograms contained in these statistics can
8.747+also be useful in some cases for more fine-grained nullability\\list length
8.748+filter pushdown.
8.749+"
8.750+ :initarg :size-statistics :initform nil :type
8.751+ (or null dat/parquet:parquet-size-statistics)))
8.752+ (:documentation "Description for column metadata
8.753+"))
8.754+(defclass dat/parquet:parquet-encryption-with-footer-key
8.755+ (dat/parquet:parquet-object) nil)
8.756+(defclass dat/parquet:parquet-encryption-with-column-key
8.757+ (dat/parquet:parquet-object)
8.758+ ((dat/parquet::path-in-schema :documentation "Column path in schema *
8.759+"
8.760+ :initarg :path-in-schema :type (vector string))
8.761+ (dat/parquet::key-metadata :documentation
8.762+ "Retrieval metadata of column encryption key *
8.763+"
8.764+ :initarg :key-metadata :initform nil :type
8.765+ (or null std/type:octet-vector))))
8.766+(defclass dat/parquet:parquet-column-crypto-meta-data
8.767+ (dat/parquet:parquet-object)
8.768+ ((dat/parquet::encryption-with-footer-key :initarg
8.769+ :encryption-with-footer-key :initform nil :type
8.770+ (or null dat/parquet:parquet-encryption-with-footer-key))
8.771+ (dat/parquet::encryption-with-column-key :initarg
8.772+ :encryption-with-column-key :initform nil :type
8.773+ (or null dat/parquet:parquet-encryption-with-column-key))))
8.774+(defclass dat/parquet:parquet-column-chunk (dat/parquet:parquet-object)
8.775+ ((dat/parquet::file-path :documentation
8.776+ "File where column data is stored. If not set, assumed to be same file as
8.777+metadata. This path is relative to the current file.
8.778+
8.779+"
8.780+ :initarg :file-path :initform nil :type (or null string))
8.781+ (dat/parquet::file-offset :documentation
8.782+ "Deprecated: Byte offset in file_path to the ColumnMetaData
8.783+
8.784+Past use of this field has been inconsistent, with some implementations
8.785+using it to point to the ColumnMetaData and some using it to point to
8.786+the first page in the column chunk. In many cases, the ColumnMetaData at this
8.787+location is wrong. This field is now deprecated and should not be used.
8.788+Writers should set this field to 0 if no ColumnMetaData has been written outside
8.789+the footer.
8.790+"
8.791+ :initarg :file-offset :type (signed-byte 64))
8.792+ (dat/parquet::meta-data :documentation
8.793+ "Column metadata for this chunk. Some writers may also replicate this at the
8.794+location pointed to by file_path\\file_offset.
8.795+Note: while marked as optional, this field is in fact required by most major
8.796+Parquet implementations. As such, writers MUST populate this field.
8.797+
8.798+"
8.799+ :initarg :meta-data :initform nil :type
8.800+ (or null dat/parquet:parquet-column-meta-data))
8.801+ (dat/parquet::offset-index-offset :documentation
8.802+ "File offset of ColumnChunk's OffsetIndex *
8.803+"
8.804+ :initarg :offset-index-offset :initform nil :type
8.805+ (or null (signed-byte 64)))
8.806+ (dat/parquet::offset-index-length :documentation
8.807+ "Size of ColumnChunk's OffsetIndex, in bytes *
8.808+"
8.809+ :initarg :offset-index-length :initform nil :type
8.810+ (or null (signed-byte 32)))
8.811+ (dat/parquet::column-index-offset :documentation
8.812+ "File offset of ColumnChunk's ColumnIndex *
8.813+"
8.814+ :initarg :column-index-offset :initform nil :type
8.815+ (or null (signed-byte 64)))
8.816+ (dat/parquet::column-index-length :documentation
8.817+ "Size of ColumnChunk's ColumnIndex, in bytes *
8.818+"
8.819+ :initarg :column-index-length :initform nil :type
8.820+ (or null (signed-byte 32)))
8.821+ (dat/parquet::crypto-metadata :documentation
8.822+ "Crypto metadata of encrypted columns *
8.823+"
8.824+ :initarg :crypto-metadata :initform nil :type
8.825+ (or null dat/parquet:parquet-column-crypto-meta-data))
8.826+ (dat/parquet::encrypted-column-metadata :documentation
8.827+ "Encrypted column metadata for this chunk *
8.828+"
8.829+ :initarg :encrypted-column-metadata :initform nil :type
8.830+ (or null std/type:octet-vector))))
8.831+(defclass dat/parquet:parquet-row-group (dat/parquet:parquet-object)
8.832+ ((dat/parquet::columns :documentation
8.833+ "Metadata for each column chunk in this row group.
8.834+This list must have the same order as the SchemaElement list in FileMetaData.
8.835+
8.836+"
8.837+ :initarg :columns :type (vector dat/parquet:parquet-column-chunk))
8.838+ (dat/parquet::total-byte-size :documentation
8.839+ "Total byte size of all the uncompressed column data in this row group *
8.840+"
8.841+ :initarg :total-byte-size :type (signed-byte 64))
8.842+ (dat/parquet::num-rows :documentation
8.843+ "Number of rows in this row group *
8.844+"
8.845+ :initarg :num-rows :type (signed-byte 64))
8.846+ (dat/parquet::sorting-columns :documentation
8.847+ "If set, specifies a sort ordering of the rows in this RowGroup.
8.848+The sorting columns can be a subset of all the columns.
8.849+"
8.850+ :initarg :sorting-columns :initform nil :type
8.851+ (or null (vector dat/parquet:parquet-sorting-column)))
8.852+ (dat/parquet::file-offset :documentation
8.853+ "Byte offset from beginning of file to first page (data or dictionary)
8.854+in this row group *
8.855+"
8.856+ :initarg :file-offset :initform nil :type
8.857+ (or null (signed-byte 64)))
8.858+ (dat/parquet::total-compressed-size :documentation
8.859+ "Total byte size of all compressed (and potentially encrypted) column data
8.860+in this row group *
8.861+"
8.862+ :initarg :total-compressed-size :initform nil :type
8.863+ (or null (signed-byte 64)))
8.864+ (dat/parquet::ordinal :documentation "Row group ordinal in the file *
8.865+"
8.866+ :initarg :ordinal :initform nil :type (or null (signed-byte 16)))))
8.867+(defclass dat/parquet:parquet-type-defined-order (dat/parquet:parquet-object)
8.868+ nil
8.869+ (:documentation
8.870+ "Empty struct to signal the order defined by the physical or logical type
8.871+"))
8.872+(defclass dat/parquet:parquet-column-order (dat/parquet:parquet-object)
8.873+ ((dat/parquet::type-order :documentation
8.874+ "The sort orders for logical types are:
8.875+ UTF8 - unsigned byte-wise comparison
8.876+ INT8 - signed comparison
8.877+ INT16 - signed comparison
8.878+ INT32 - signed comparison
8.879+ INT64 - signed comparison
8.880+ UINT8 - unsigned comparison
8.881+ UINT16 - unsigned comparison
8.882+ UINT32 - unsigned comparison
8.883+ UINT64 - unsigned comparison
8.884+ DECIMAL - signed comparison of the represented value
8.885+ DATE - signed comparison
8.886+ TIME_MILLIS - signed comparison
8.887+ TIME_MICROS - signed comparison
8.888+ TIMESTAMP_MILLIS - signed comparison
8.889+ TIMESTAMP_MICROS - signed comparison
8.890+ INTERVAL - undefined
8.891+ JSON - unsigned byte-wise comparison
8.892+ BSON - unsigned byte-wise comparison
8.893+ ENUM - unsigned byte-wise comparison
8.894+ LIST - undefined
8.895+ MAP - undefined
8.896+
8.897+In the absence of logical types, the sort order is determined by the physical type:
8.898+ BOOLEAN - false, true
8.899+ INT32 - signed comparison
8.900+ INT64 - signed comparison
8.901+ INT96 (only used for legacy timestamps) - undefined
8.902+ FLOAT - signed comparison of the represented value (*)
8.903+ DOUBLE - signed comparison of the represented value (*)
8.904+ BYTE_ARRAY - unsigned byte-wise comparison
8.905+ FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison
8.906+
8.907+(*) Because the sorting order is not specified properly for floating
8.908+ point values (relations vs. total ordering) the following
8.909+ compatibility rules should be applied when reading statistics:
8.910+ - If the min is a NaN, it should be ignored.
8.911+ - If the max is a NaN, it should be ignored.
8.912+ - If the min is +0, the row group may contain -0 values as well.
8.913+ - If the max is -0, the row group may contain +0 values as well.
8.914+ - When looking for NaN values, min and max should be ignored.
8.915+
8.916+ When writing statistics the following rules should be followed:
8.917+ - NaNs should not be written to min or max statistics fields.
8.918+ - If the computed max value is zero (whether negative or positive),
8.919+ `+0.0` should be written into the max statistics field.
8.920+ - If the computed min value is zero (whether negative or positive),
8.921+ `-0.0` should be written into the min statistics field.
8.922+"
8.923+ :initarg :type-order :initform nil :type
8.924+ (or null dat/parquet:parquet-type-defined-order)))
8.925+ (:documentation
8.926+ "Union to specify the order used for the min_value and max_value fields for a
8.927+column. This union takes the role of an enhanced enum that allows rich
8.928+elements (which will be needed for a collation-based ordering in the future).
8.929+
8.930+Possible values are:
8.931+* TypeDefinedOrder - the column uses the order defined by its logical or
8.932+ physical type (if there is no logical type).
8.933+
8.934+If the reader does not support the value of this union, min and max stats
8.935+for this column should be ignored.
8.936+"))
8.937+(defclass dat/parquet:parquet-page-location (dat/parquet:parquet-object)
8.938+ ((dat/parquet::offset :documentation "Offset of the page in the file *
8.939+"
8.940+ :initarg :offset :type (signed-byte 64))
8.941+ (dat/parquet::compressed-page-size :documentation
8.942+ "Size of the page, including header. Sum of compressed_page_size and header
8.943+length
8.944+"
8.945+ :initarg :compressed-page-size :type (signed-byte 32))
8.946+ (dat/parquet::first-row-index :documentation
8.947+ "Index within the RowGroup of the first row of the page. When an
8.948+OffsetIndex is present, pages must begin on row boundaries
8.949+(repetition_level = 0).
8.950+"
8.951+ :initarg :first-row-index :type (signed-byte 64))))
8.952+(defclass dat/parquet:parquet-offset-index (dat/parquet:parquet-object)
8.953+ ((dat/parquet::page-locations :documentation
8.954+ "PageLocations, ordered by increasing PageLocation.offset. It is required
8.955+that page_locations[i].first_row_index < page_locations[i+1].first_row_index.
8.956+"
8.957+ :initarg :page-locations :type
8.958+ (vector dat/parquet:parquet-page-location))
8.959+ (dat/parquet::unencoded-byte-array-data-bytes :documentation
8.960+ "Unencoded\\uncompressed size for BYTE_ARRAY types.
8.961+
8.962+See documention for unencoded_byte_array_data_bytes in SizeStatistics for
8.963+more details on this field.
8.964+"
8.965+ :initarg :unencoded-byte-array-data-bytes :initform nil :type
8.966+ (or null (vector (signed-byte 64)))))
8.967+ (:documentation "Optional offsets for each data page in a ColumnChunk.
8.968+
8.969+Forms part of the page index, along with ColumnIndex.
8.970+
8.971+OffsetIndex may be present even if ColumnIndex is not.
8.972+"))
8.973+(defclass dat/parquet:parquet-column-index (dat/parquet:parquet-object)
8.974+ ((dat/parquet::null-pages :documentation
8.975+ "A list of Boolean values to determine the validity of the corresponding
8.976+min and max values. If true, a page contains only null values, and writers
8.977+have to set the corresponding entries in min_values and max_values to
8.978+byte[0], so that all lists have the same length. If false, the
8.979+corresponding entries in min_values and max_values must be valid.
8.980+"
8.981+ :initarg :null-pages :type (vector boolean))
8.982+ (dat/parquet::min-values :documentation
8.983+ "Two lists containing lower and upper bounds for the values of each page
8.984+determined by the ColumnOrder of the column. These may be the actual
8.985+minimum and maximum values found on a page, but can also be (more compact)
8.986+values that do not exist on a page. For example, instead of storing \\\\Blart
8.987+Versenwald III\\, a writer may set min_values[i]=\\B\\, max_values[i]=\\C\\.
8.988+Such more compact values must still be valid values within the column's
8.989+logical type. Readers must make sure that list entries are populated before
8.990+using them by inspecting null_pages.
8.991+"
8.992+ :initarg :min-values :type (vector std/type:octet-vector))
8.993+ (dat/parquet::max-values :initarg :max-values :type
8.994+ (vector std/type:octet-vector))
8.995+ (dat/parquet::boundary-order :documentation
8.996+ "Stores whether both min_values and max_values are ordered and if so, in
8.997+which direction. This allows readers to perform binary searches in both
8.998+lists. Readers cannot assume that max_values[i] <= min_values[i+1], even
8.999+if the lists are ordered.
8.1000+"
8.1001+ :initarg :boundary-order :type dat/parquet::parquet-boundary-order)
8.1002+ (dat/parquet::null-counts :documentation
8.1003+ "A list containing the number of null values for each page *
8.1004+"
8.1005+ :initarg :null-counts :initform nil :type
8.1006+ (or null (vector (signed-byte 64))))
8.1007+ (dat/parquet::repetition-level-histograms :documentation
8.1008+ "Contains repetition level histograms for each page
8.1009+concatenated together. The repetition_level_histogram field on
8.1010+SizeStatistics contains more details.
8.1011+
8.1012+When present the length should always be (number of pages *
8.1013+(max_repetition_level + 1)) elements.
8.1014+
8.1015+Element 0 is the first element of the histogram for the first page.
8.1016+Element (max_repetition_level + 1) is the first element of the histogram
8.1017+for the second page.
8.1018+
8.1019+"
8.1020+ :initarg :repetition-level-histograms :initform nil :type
8.1021+ (or null (vector (signed-byte 64))))
8.1022+ (dat/parquet::definition-level-histograms :documentation
8.1023+ "Same as repetition_level_histograms except for definitions levels.
8.1024+
8.1025+"
8.1026+ :initarg :definition-level-histograms :initform nil :type
8.1027+ (or null (vector (signed-byte 64)))))
8.1028+ (:documentation
8.1029+ "Optional statistics for each data page in a ColumnChunk.
8.1030+
8.1031+Forms part the page index, along with OffsetIndex.
8.1032+
8.1033+If this structure is present, OffsetIndex must also be present.
8.1034+
8.1035+For each field in this structure, <field>[i] refers to the page at
8.1036+OffsetIndex.page_locations[i]
8.1037+"))
8.1038+(defclass dat/parquet:parquet-aes-gcm-v1 (dat/parquet:parquet-object)
8.1039+ ((dat/parquet::aad-prefix :documentation "AAD prefix *
8.1040+"
8.1041+ :initarg :aad-prefix :initform nil :type
8.1042+ (or null std/type:octet-vector))
8.1043+ (dat/parquet::aad-file-unique :documentation
8.1044+ "Unique file identifier part of AAD suffix *
8.1045+"
8.1046+ :initarg :aad-file-unique :initform nil :type
8.1047+ (or null std/type:octet-vector))
8.1048+ (dat/parquet::supply-aad-prefix :documentation
8.1049+ "In files encrypted with AAD prefix without storing it,
8.1050+readers must supply the prefix *
8.1051+"
8.1052+ :initarg :supply-aad-prefix :initform nil :type (or null boolean))))
8.1053+(defclass dat/parquet:parquet-aes-gcm-ctr-v1 (dat/parquet:parquet-object)
8.1054+ ((dat/parquet::aad-prefix :documentation "AAD prefix *
8.1055+"
8.1056+ :initarg :aad-prefix :initform nil :type
8.1057+ (or null std/type:octet-vector))
8.1058+ (dat/parquet::aad-file-unique :documentation
8.1059+ "Unique file identifier part of AAD suffix *
8.1060+"
8.1061+ :initarg :aad-file-unique :initform nil :type
8.1062+ (or null std/type:octet-vector))
8.1063+ (dat/parquet::supply-aad-prefix :documentation
8.1064+ "In files encrypted with AAD prefix without storing it,
8.1065+readers must supply the prefix *
8.1066+"
8.1067+ :initarg :supply-aad-prefix :initform nil :type (or null boolean))))
8.1068+(defclass dat/parquet:parquet-encryption-algorithm (dat/parquet:parquet-object)
8.1069+ ((dat/parquet::aes-gcm-v1 :initarg :aes-gcm-v1 :initform nil :type
8.1070+ (or null dat/parquet:parquet-aes-gcm-v1))
8.1071+ (dat/parquet::aes-gcm-ctr-v1 :initarg :aes-gcm-ctr-v1 :initform nil
8.1072+ :type (or null dat/parquet:parquet-aes-gcm-ctr-v1))))
8.1073+(defclass dat/parquet:parquet-file-meta-data (dat/parquet:parquet-object)
8.1074+ ((dat/parquet::version :documentation "Version of this file *
8.1075+"
8.1076+ :initarg :version :type (signed-byte 32))
8.1077+ (dat/parquet::schema :documentation
8.1078+ "Parquet schema for this file. This schema contains metadata for all the columns.
8.1079+The schema is represented as a tree with a single root. The nodes of the tree
8.1080+are flattened to a list by doing a depth-first traversal.
8.1081+The column metadata contains the path in the schema for that column which can be
8.1082+used to map columns to nodes in the schema.
8.1083+The first element is the root *
8.1084+"
8.1085+ :initarg :schema :type (vector dat/parquet:parquet-schema-element))
8.1086+ (dat/parquet::num-rows :documentation "Number of rows in this file *
8.1087+"
8.1088+ :initarg :num-rows :type (signed-byte 64))
8.1089+ (dat/parquet::row-groups :documentation "Row groups in this file *
8.1090+"
8.1091+ :initarg :row-groups :type (vector dat/parquet:parquet-row-group))
8.1092+ (dat/parquet::key-value-metadata :documentation
8.1093+ "Optional key\\value metadata *
8.1094+"
8.1095+ :initarg :key-value-metadata :initform nil :type
8.1096+ (or null (vector dat/parquet:parquet-key-value)))
8.1097+ (dat/parquet::created-by :documentation
8.1098+ "String for application that wrote this file. This should be in the format
8.1099+<Application> version <App Version> (build <App Build Hash>).
8.1100+e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
8.1101+
8.1102+"
8.1103+ :initarg :created-by :initform nil :type (or null string))
8.1104+ (dat/parquet::column-orders :documentation
8.1105+ "Sort order used for the min_value and max_value fields in the Statistics
8.1106+objects and the min_values and max_values fields in the ColumnIndex
8.1107+objects of each column in this file. Sort orders are listed in the order
8.1108+matching the columns in the schema. The indexes are not necessary the same
8.1109+though, because only leaf nodes of the schema are represented in the list
8.1110+of sort orders.
8.1111+
8.1112+Without column_orders, the meaning of the min_value and max_value fields
8.1113+in the Statistics object and the ColumnIndex object is undefined. To ensure
8.1114+well-defined behaviour, if these fields are written to a Parquet file,
8.1115+column_orders must be written as well.
8.1116+
8.1117+The obsolete min and max fields in the Statistics object are always sorted
8.1118+by signed comparison regardless of column_orders.
8.1119+"
8.1120+ :initarg :column-orders :initform nil :type
8.1121+ (or null (vector dat/parquet:parquet-column-order)))
8.1122+ (dat/parquet::encryption-algorithm :documentation
8.1123+ "Encryption algorithm. This field is set only in encrypted files
8.1124+with plaintext footer. Files with encrypted footer store algorithm id
8.1125+in FileCryptoMetaData structure.
8.1126+"
8.1127+ :initarg :encryption-algorithm :initform nil :type
8.1128+ (or null dat/parquet:parquet-encryption-algorithm))
8.1129+ (dat/parquet::footer-signing-key-metadata :documentation
8.1130+ "Retrieval metadata of key used for signing the footer.
8.1131+Used only in encrypted files with plaintext footer.
8.1132+"
8.1133+ :initarg :footer-signing-key-metadata :initform nil :type
8.1134+ (or null std/type:octet-vector)))
8.1135+ (:documentation "Description for file metadata
8.1136+"))
8.1137+(defclass dat/parquet:parquet-file-crypto-meta-data
8.1138+ (dat/parquet:parquet-object)
8.1139+ ((dat/parquet::encryption-algorithm :documentation
8.1140+ "Encryption algorithm. This field is only used for files
8.1141+with encrypted footer. Files with plaintext footer store algorithm id
8.1142+inside footer (FileMetaData structure).
8.1143+"
8.1144+ :initarg :encryption-algorithm :type
8.1145+ dat/parquet:parquet-encryption-algorithm)
8.1146+ (dat/parquet::key-metadata :documentation
8.1147+ "Retrieval metadata of key used for encryption of footer,
8.1148+and (possibly) columns *
8.1149+"
8.1150+ :initarg :key-metadata :initform nil :type
8.1151+ (or null std/type:octet-vector)))
8.1152+ (:documentation "Crypto metadata for files with encrypted footer *
8.1153+"))
9.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
9.2+++ b/lisp/lib/rt/fuzz.lisp Mon Sep 02 18:31:19 2024 -0400
9.3@@ -0,0 +1,51 @@
9.4+;;; fuzz.lisp --- RT Fuzz
9.5+
9.6+;; FUZZER API
9.7+
9.8+;;; Commentary:
9.9+
9.10+;;
9.11+;; wiki: https://en.wikipedia.org/wiki/Fuzzing
9.12+
9.13+;;; Code:
9.14+(in-package :rt/fuzz)
9.15+
9.16+(defvar *default-fuzz-generator*
9.17+ (lambda (state)
9.18+ (random most-positive-fixnum state)))
9.19+
9.20+(defclass fuzzer ()
9.21+ ((state :initform (make-random-state t)
9.22+ :initarg :state
9.23+ :accessor fuzz-state)
9.24+ (generator :initform *default-fuzz-generator*
9.25+ :initarg :generator
9.26+ :type function
9.27+ :accessor fuzz-generator))
9.28+ (:documentation "An object which provides invalid, unexpected or random data as inputs to some
9.29+program."))
9.30+
9.31+(defgeneric fuzz (self &key &allow-other-keys)
9.32+ (:method ((self fuzzer) &key &allow-other-keys)
9.33+ (funcall (the function (fuzz-generator self)) (fuzz-state self)))
9.34+ (:method ((self fuzzer) &key count)
9.35+ (if count
9.36+ (let ((ret))
9.37+ (dotimes (i count ret)
9.38+ (push (funcall (the function (fuzz-generator self)) (fuzz-state self)) ret)))
9.39+ (fuzz self))))
9.40+
9.41+(defgeneric fuzz* (state generator &key &allow-other-keys)
9.42+ (:method ((state list) (generator function) &key (count 1))
9.43+ (let ((ret))
9.44+ (dotimes (i count ret)
9.45+ (push (funcall generator state) ret))))
9.46+ (:method ((state vector) (generator function) &key (count 1))
9.47+ (let ((ret (make-array count :fill-pointer 0)))
9.48+ (dotimes (i count ret)
9.49+ (setf (aref ret i) (funcall generator state)))))
9.50+ (:method ((state hash-table) (generator function) &key (count 1))
9.51+ (let ((ret (make-hash-table)))
9.52+ (dotimes (i count ret)
9.53+ (destructuring-bind (k v) (funcall generator state)
9.54+ (setf (gethash k ret) v))))))
