1.1--- a/core/lib/alch.org Sat Jun 22 23:54:39 2024 -0400
1.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
1.3@@ -1,1 +0,0 @@
1.4-{{{header(alch,Richard Westhaver,ellis@rwest.io)}}}
2.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
2.2+++ b/core/lib/aud.org Sat Jun 22 23:55:21 2024 -0400
2.3@@ -0,0 +1,6 @@
2.4+#+title: aud
2.5+#+author: Richard Westhaver
2.6+#+email: richard.westhaver@gmail.com
2.7+#+description: High-level Audio Library
2.8+#+filetags: lisp media audio
2.9+#+setupfile: ../../../clean.theme
3.1--- a/core/lib/audio.org Sat Jun 22 23:54:39 2024 -0400
3.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
3.3@@ -1,1 +0,0 @@
3.4-{{{header(audio,Richard Westhaver,ellis@rwest.io)}}}
4.1--- /dev/null Thu Jan 01 00:00:00 1970 +0000
4.2+++ b/core/lib/cli.org Sat Jun 22 23:55:21 2024 -0400
4.3@@ -0,0 +1,127 @@
4.4+#+title: CLI
4.5+#+author: Richard Westhaver
4.6+#+description: Command-line Interface Library
4.7+#+setupfile: clean.theme
4.8+#+filetags: lisp cli
4.9+#+property: header-args :dir ~/comp/core/ :exports both
4.10+#+setupfile: ../../../clean.theme
4.11+#+INFOJS_OPT: toc:nil
4.12+
4.13+* Overview
4.14+The =cli= library is a fully-featured toolkit for building [[https://en.wikipedia.org/wiki/Command-line_interface][CLI]]
4.15+programs.
4.16+
4.17+
4.18+* stats
4.19+** dependencies
4.20+#+begin_src lisp :results list
4.21+ (ql:quickload :doc)
4.22+ (let ((sys (asdf:find-system :cli)))
4.23+ (asdf:system-depends-on sys))
4.24+#+end_src
4.25+
4.26+#+RESULTS:
4.27+- std
4.28+- log
4.29+
4.30+** dependents
4.31+#+begin_src lisp :results list
4.32+ (ql:quickload :doc)
4.33+ (let ((sys (doc:system-documentation :cli)))
4.34+ (mapcar (lambda (s) (asdf:component-name (doc:doc-system s))) (doc:doc-dependents sys)))
4.35+#+end_src
4.36+
4.37+#+RESULTS:
4.38+- prelude
4.39+- pod
4.40+- packy
4.41+- vc
4.42+- skel
4.43+- obj
4.44+- cli/tests
4.45+- user
4.46+
4.47+** files
4.48+#+begin_src lisp :output replace :results list
4.49+ (mapcar (lambda (p)
4.50+ (enough-namestring p (asdf:system-source-directory :cli)))
4.51+ (doc:doc-files (doc:system-documentation :cli)))
4.52+#+end_src
4.53+
4.54+#+RESULTS:
4.55+- pkg.lisp
4.56+- ansi.lisp
4.57+- env.lisp
4.58+- progress.lisp
4.59+- spark.lisp
4.60+- repl.lisp
4.61+- shell.lisp
4.62+- prompt.lisp
4.63+- ed.lisp
4.64+- tools/tmux.lisp
4.65+- tools/pacman.lisp
4.66+- clap/pkg.lisp
4.67+- clap/ast.lisp
4.68+- clap/vars.lisp
4.69+- clap/util.lisp
4.70+- clap/macs.lisp
4.71+- clap/proto.lisp
4.72+- clap/opt.lisp
4.73+- clap/cmd.lisp
4.74+- clap/cli.lisp
4.75+- cli.lisp
4.76+
4.77+** packages
4.78+#+begin_src lisp :output replace :results list
4.79+ (mapcar (lambda (p) (package-name (doc:doc-package p)))
4.80+ (doc:doc-packages (doc:system-documentation :cli)))
4.81+#+end_src
4.82+
4.83+#+RESULTS:
4.84+- CLI/CLAP/OBJ
4.85+- CLI/CLAP/SIMPLE
4.86+- CLI/SHELL
4.87+- CLI/ANSI
4.88+- CLI/CLAP/PROTO
4.89+- CLI/CLAP/AST
4.90+- CLI/ED
4.91+- CLI
4.92+- CLI/CLAP
4.93+- CLI/CLAP/VARS
4.94+- CLI/TESTS
4.95+- CLI/REPL
4.96+- CLI/CLAP/UTIL
4.97+- CLI/ENV
4.98+- CLI/PROGRESS
4.99+- CLI/PROMPT
4.100+- CLI-USER
4.101+- CLI/SPARK
4.102+- CLI/TMUX
4.103+- CLI/CLAP/MACS
4.104+
4.105+** tests
4.106+#+begin_src lisp :output replace
4.107+ (ql:quickload :cli/tests)
4.108+ (print rt:*test-suite* nil)
4.109+#+end_src
4.110+
4.111+#+RESULTS:
4.112+: #<RT:TEST-SUITE CLI [10:10:0:0] {103BAFED63}>
4.113+
4.114+#+begin_src lisp :output replace
4.115+ (rt:tests rt:*test-suite*)
4.116+#+end_src
4.117+
4.118+#+RESULTS:
4.119+#+begin_example
4.120+(#<RT:TEST MAIN-OUTPUT :fn MAIN-OUTPUT-TEST2479 {103BB3F5C3}>
4.121+ #<RT:TEST CLAP-AST :fn CLAP-AST-TEST2478 {103BB3F4E3}>
4.122+ #<RT:TEST ENV :fn ENV-TEST2477 {103BB3F403}>
4.123+ #<RT:TEST REPL :fn REPL-TEST2476 {103BB3F323}>
4.124+ #<RT:TEST SPARK :fn SPARK-TEST2475 {103BB3F243}>
4.125+ #<RT:TEST PROGRESS :fn PROGRESS-TEST2474 {103BB3F163}>
4.126+ #<RT:TEST CLAP-OPTS :fn CLAP-OPTS-TEST2473 {103BB3F083}>
4.127+ #<RT:TEST CLAP-BASIC :fn CLAP-BASIC-TEST2472 {103BB3EF73}>
4.128+ #<RT:TEST CLI-PROMPT :fn CLI-PROMPT-TEST2471 {103BB10603}>
4.129+ #<RT:TEST ANSI :fn ANSI-TEST2470 {103BAFEF73}>)
4.130+#+end_example
5.1--- a/core/lib/crypto.org Sat Jun 22 23:54:39 2024 -0400
5.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
5.3@@ -1,1 +0,0 @@
5.4-{{{header(crypto,Richard Westhaver,ellis@rwest.io)}}}
6.1--- a/core/lib/db.org Sat Jun 22 23:54:39 2024 -0400
6.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
6.3@@ -1,1 +0,0 @@
6.4-{{{header(db,Richard Westhaver,ellis@rwest.io)}}}
7.1--- a/core/lib/dl.org Sat Jun 22 23:54:39 2024 -0400
7.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
7.3@@ -1,1 +0,0 @@
7.4-{{{header(dl,Richard Westhaver,ellis@rwest.io)}}}
8.1--- a/core/lib/dot.org Sat Jun 22 23:54:39 2024 -0400
8.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
8.3@@ -1,1 +0,0 @@
8.4-{{{header(dot,Richard Westhaver,ellis@rwest.io)}}}
9.1--- a/core/lib/flate.org Sat Jun 22 23:54:39 2024 -0400
9.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
9.3@@ -1,1 +0,0 @@
9.4-{{{header(flate,Richard Westhaver,ellis@rwest.io)}}}
10.1--- a/core/lib/hash.org Sat Jun 22 23:54:39 2024 -0400
10.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
10.3@@ -1,1 +0,0 @@
10.4-{{{header(hash,Richard Westhaver,ellis@rwest.io)}}}
11.1--- a/core/lib/logger.org Sat Jun 22 23:54:39 2024 -0400
11.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
11.3@@ -1,1 +0,0 @@
11.4-{{{header(logger,Richard Westhaver,ellis@rwest.io)}}}
12.1--- a/core/lib/sxp.org Sat Jun 22 23:54:39 2024 -0400
12.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
12.3@@ -1,1 +0,0 @@
12.4-{{{header(sxp,Richard Westhaver,ellis@rwest.io)}}}
13.1--- a/core/lib/tenex.org Sat Jun 22 23:54:39 2024 -0400
13.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
13.3@@ -1,1 +0,0 @@
13.4-{{{header(tenex,Richard Westhaver,ellis@rwest.io)}}}
14.1--- a/core/lib/util.org Sat Jun 22 23:54:39 2024 -0400
14.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
14.3@@ -1,1 +0,0 @@
14.4-{{{header(util,Richard Westhaver,ellis@rwest.io)}}}
15.1--- a/nas-t/notes.org Sat Jun 22 23:54:39 2024 -0400
15.2+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
15.3@@ -1,239 +0,0 @@
15.4-#+title: notes
15.5-#+author: Richard Westhaver
15.6-#+email: ellis@rwest.io
15.7-#+description: NAS-T Notes
15.8-#+setupfile: ../../clean.theme
15.9-#+BIBLIOGRAPHY: refs.bib
15.10-* File Systems
15.11-** BTRFS
15.12-#+begin_quote
15.13-BTRFS is a Linux filesystem based on copy-on-write, allowing for
15.14-efficient snapshots and clones.
15.15-
15.16-It uses B-trees as its main on-disk data structure. The design goal is
15.17-to work well for many use cases and workloads. To this end, much
15.18-effort has been directed to maintaining even performance as the
15.19-filesystem ages, rather than trying to support a particular narrow
15.20-benchmark use-case.
15.21-
15.22-Linux filesystems are installed on smartphones as well as enterprise
15.23-servers. This entails challenges on many different fronts.
15.24-
15.25-- Scalability :: The filesystem must scale in many dimensions: disk
15.26- space, memory, and CPUs.
15.27-
15.28-- Data integrity :: Losing data is not an option, and much effort is
15.29- expended to safeguard the content. This includes checksums, metadata
15.30- duplication, and RAID support built into the filesystem.
15.31-
15.32-- Disk diversity :: The system should work well with SSDs and hard
15.33- disks. It is also expected to be able to use an array of different
15.34- sized disks, which poses challenges to the RAID and striping
15.35- mechanisms.
15.36-#+end_quote
15.37--- [cite/t/f:@btrfs]
15.38-*** [2023-08-08 Tue] btrfs performance speculation ::
15.39- - [[https://www.percona.com/blog/taking-a-look-at-btrfs-for-mysql/]]
15.40- - zfs outperforms immensely, but potential misconfiguration on btrfs side (virt+cow
15.41- still enabled?)
15.42- - https://www.ctrl.blog/entry/btrfs-vs-ext4-performance.html
15.43- - see the follow up comment on this post
15.44- - https://www.reddit.com/r/archlinux/comments/o2gc42/is_the_performance_hit_of_btrfs_serious_is_it/
15.45- #+begin_quote
15.46- I’m the author of OP’s first link. I use BtrFS today. I often shift lots of
15.47- de-duplicatable data around, and benefit greatly from file cloning. The data is actually
15.48- the same data that caused the slow performance in the article. BtrFS and file cloning
15.49- now performs this task quicker than a traditional file system. (Hm. It’s time for a
15.50- follow-up article.)
15.51-
15.52- In a laptop with one drive: it doesn’t matter too much unless you do work that benefit
15.53- from file cloning or snapshots. This will likely require you to adjust your tooling and
15.54- workflow. I’ve had to rewrite the software I use every day to make it take advantage of
15.55- the capabilities of a more modern file system. You won’t benefit much from the data
15.56- recovery and redundancy features unless you’ve got two storage drives in your laptop and
15.57- can setup redundant data copies.
15.58-
15.59- on similar hardware to mine?
15.60-
15.61- It’s not a question about your hardware as much as how you use it. The bad performance I
15.62- documented was related to lots and lots of simultaneous random reads and writes. This
15.63- might not be representative of how you use your computer.
15.64- #+end_quote
15.65- - https://dl.acm.org/doi/fullHtml/10.1145/3386362
15.66- - this is about distributed file systems (in this case Ceph) - they argue against
15.67- basing DFS on ondisk-format filesystems (XFS ext4) - developed BlueStore as
15.68- backend, which runs directly on raw storage hardware.
15.69- - this is a good approach, but expensive (2 years in development) and risky
15.70- - better approach is to take advantage of a powerful enough existing ondisk-FS
15.71- format and pair it with supporting modules which abstract away the 'distributed'
15.72- mechanics.
15.73- - the strategy presented here is critical for enterprise-grade hardware where the
15.74- ondisk filesystem becomes the bottleneck that you're looking to optimize
15.75- - https://lore.kernel.org/lkml/cover.1676908729.git.dsterba@suse.com/
15.76- - linux 6.3 patch by David Sterba [2023-02-20 Mon]
15.77- - btrfs continues to show improvements in the linux kernel, ironing out the kinks
15.78- - makes it hard to compare benchmarks tho :/
15.79-*** MacOS support
15.80-- see this WIP k-ext for macos: [[https://github.com/relalis/macos-btrfs][macos-btrfs]]
15.81- - maybe we can help out with the VFS/mount support
15.82-*** on-disk format
15.83-- [[https://btrfs.readthedocs.io/en/latest/dev/On-disk-format.html][on-disk-format]]
15.84-- 'btrfs consists entirely of several trees. the trees use copy-on-write.'
15.85-- trees are stored in nodes which belong to a level in the b-tree structure.
15.86-- internal nodes (inodes) contain refs to other inodes on the /next/ level OR
15.87- - to leaf nodes then the level reaches 0.
15.88-- leaf nodes contain various types depending on the tree.
15.89-- basic structures
15.90- - 0:8 uint = objectid, each tree has its own set of object IDs
15.91- - 8:1 uint = item type
15.92- - 9:8 uint = offset, depends on type.
15.93- - little-endian
15.94- - fields are unsigned
15.95- - *superblock*
15.96- - primary superblock is located at 0x10000 (64KiB)
15.97- - Mirror copies of the superblock are located at physical addresses 0x4000000 (64
15.98- MiB) and 0x4000000000 (256GiB), if valid. copies are updated simultaneously.
15.99- - during mount only the first super block at 0x10000 is read, error causes mount to
15.100- fail.
15.101- - BTRFS onls recognizes disks with a valid 0x10000 superblock.
15.102- - *header*
15.103- - stored at the start of every inode
15.104- - data following it depends on whether it is an internal or leaf node.
15.105- - *inode*
15.106- - node header followed by a number of key pointers
15.107- - 0:11 key
15.108- - 11:8 uint = block number
15.109- - 19:8 uint = generation
15.110- - *lnode*
15.111- - leaf nodes contain header followed by key pointers
15.112- - 0:11 key
15.113- - 11:4 uint = data offset relative to end of header(65)
15.114- - 15:4 uint = data size
15.115-- objects
15.116- - ROOT_TREE
15.117- - holds ROOT_ITEMs, ROOT_REFs, and ROOT_BACKREFs for every tree other than itself.
15.118- - used to find the other trees and to determine the subvol structure.
15.119- - holds items for the 'root tree directory'. laddr is store in the superblock
15.120- - objectIDs
15.121- - free ids: BTRFS_FIRST_FREE_OBJECTID=256ULL:BTRFS_LAST_FREE_OBJECTID=-256ULL
15.122- - otherwise used for internal use
15.123-*** send-stream format
15.124-- [[https://btrfs.readthedocs.io/en/latest/dev/dev-send-stream.html][send stream format]]
15.125-- Send stream format represents a linear sequence of commands describing actions to be
15.126- performed on the target filesystem (receive side), created on the source filesystem
15.127- (send side).
15.128-- The stream is currently used in two ways: to generate a stream representing a
15.129- standalone subvolume (full mode) or a difference between two snapshots of the same
15.130- subvolume (incremental mode).
15.131-- The stream can be generated using a set of other subvolumes to look for extent
15.132- references that could lead to a more efficient stream by transferring only the
15.133- references and not full data.
15.134-- The stream format is abstracted from on-disk structures (though it may share some
15.135- BTRFS specifics), the stream instructions could be generated by other means than the
15.136- send ioctl.
15.137-- it's a checksum+TLV
15.138-- header: u32len,u16cmd,u32crc32c
15.139-- data: type,length,raw data
15.140-- the v2 protocol supports the encoded commands
15.141-- the commands are kinda clunky - need to MKFIL/MKDIR then RENAM to create
15.142-*** [2023-08-09 Wed] ioctls
15.143-- magic#: 0x94
15.144- - https://docs.kernel.org/userspace-api/ioctl/ioctl-number.html
15.145- - Btrfs filesystem some lifted to vfs/generic
15.146- - fs/btrfs/ioctl.h and linux/fs.h
15.147-** ZFS
15.148--- [cite/t/f:@zfs]
15.149-
15.150-- core component of TrueNAS software
15.151-** TMPFS
15.152--- [cite/t/f:@tmpfs]
15.153-- in-mem FS
15.154-** EXT4
15.155--- [cite/t/f:@ext4]
15.156-** XFS
15.157--- [cite/t/f:@xfs]
15.158--- [cite/t/f:@xfs-scalability]
15.159-* Storage Mediums
15.160-** HDD
15.161--- [cite/t/f:@hd-failure-ml]
15.162-** SSD
15.163--- [cite/t/f:@smart-ssd-qp]
15.164--- [cite/t/f:@ssd-perf-opt]
15.165-
15.166-** Flash
15.167--- [cite/t/f:@flash-openssd-systems]
15.168-** NVMe
15.169--- [cite/t/f:@nvme-ssd-ux]
15.170--- [[https://nvmexpress.org/specifications/][specifications]]
15.171-*** ZNS
15.172--- [cite/t/f:@zns-usenix]
15.173-#+begin_quote
15.174-Zoned Storage is an open source, standards-based initiative to enable data centers to
15.175-scale efficiently for the zettabyte storage capacity era. There are two technologies
15.176-behind Zoned Storage, Shingled Magnetic Recording (SMR) in ATA/SCSI HDDs and Zoned
15.177-Namespaces (ZNS) in NVMe SSDs.
15.178-#+end_quote
15.179--- [[https://zonedstorage.io/][zonedstorage.io]]
15.180--- $465 8tb 2.5"? [[https://www.serversupply.com/SSD/PCI-E/7.68TB/WESTERN%20DIGITAL/WUS4BB076D7P3E3_332270.htm][retail]]
15.181-** eMMC
15.182--- [cite/t/f:@emmc-mobile-io]
15.183-* Linux
15.184-** syscalls
15.185-*** ioctl
15.186-- [[https://elixir.bootlin.com/linux/latest/source/Documentation/userspace-api/ioctl/ioctl-number.rst][ioctl-numbers]]
15.187-* Rust
15.188-** crates
15.189-*** nix
15.190-- [[https://crates.io/crates/nix][crates.io]]
15.191-*** memmap2
15.192-- [[https://crates.io/crates/memmap2][crates.io]]
15.193-*** zstd
15.194-- [[https://crates.io/crates/zstd][crates.io]]
15.195-*** rocksdb
15.196-- [[https://crates.io/crates/rocksdb][crates.io]]
15.197-*** tokio :tokio:
15.198-- [[https://crates.io/crates/tokio][crates.io]]
15.199-*** tracing :tokio:
15.200-- [[https://crates.io/crates/tracing][crates.io]]
15.201-**** tracing-subscriber
15.202-- [[https://crates.io/crates/tracing-subscriber][crates.io]]
15.203-*** axum :tokio:
15.204-- [[https://crates.io/crates/axum][crates.io]]
15.205-*** tower :tokio:
15.206-- [[https://crates.io/crates/tower][crates.io]]
15.207-*** uuid
15.208-- [[https://crates.io/crates/uuid][crates.io]]
15.209-** unstable
15.210-*** lazy_cell
15.211-- [[https://github.com/rust-lang/rust/issues/109736][tracking-issue]]
15.212-*** {BTreeMap,BTreeSet}::extract_if
15.213-- [[https://github.com/rust-lang/rust/issues/70530][tracking-issue]]
15.214-* Lisp
15.215-** ASDF
15.216-- [[https://gitlab.common-lisp.net/asdf/asdf][gitlab.common-lisp.net]]
15.217-- [[https://asdf.common-lisp.dev/][common-lisp.dev]]
15.218-- [[https://github.com/fare/asdf/blob/master/doc/best_practices.md][best-practices]]
15.219-- includes UIOP
15.220-** Reference Projects
15.221-*** StumpWM
15.222-- [[https://github.com/stumpwm/stumpwm][github]]
15.223-*** Nyxt
15.224-- [[https://github.com/atlas-engineer/nyxt][github]]
15.225-*** Kons-9
15.226-- [[https://github.com/kaveh808/kons-9][github]]
15.227-*** cl-torrents
15.228-- [[https://github.com/vindarel/cl-torrents][github]]
15.229-*** Mezzano
15.230-- [[https://github.com/froggey/Mezzano][github]]
15.231-*** yalo
15.232-- [[https://github.com/whily/yalo][github]]
15.233-*** cl-ledger
15.234-- [[https://github.com/ledger/cl-ledger][github]]
15.235-*** Lem
15.236-- [[https://github.com/lem-project/lem][github]]
15.237-*** kindista
15.238-- [[https://github.com/kindista/kindista][github]]
15.239-*** lisp-chat
15.240-- [[https://github.com/ryukinix/lisp-chat][github]]
15.241-* Refs
15.242-#+print_bibliography:
