# HG changeset patch # User ellis # Date 1700883547 18000 # Node ID 27795b18326b278c0e12efabd2b92a20d0542511 # Parent 06842841bf425a5e2eb80086ea6361ee81d28ead blog update diff -r 06842841bf42 -r 27795b18326b .hgsubstate --- a/.hgsubstate Mon Nov 20 21:13:44 2023 -0500 +++ b/.hgsubstate Fri Nov 24 22:39:07 2023 -0500 @@ -1,2 +1,2 @@ -ae3d87d236cfb817b50cc8e41614df11a872c1d2 blog -1c57e053f043e7f164e18edda6edfc235662cadb docs +5747098f52b4e128741024c159ea5af83ef85485 blog +bb51c61e4d4b800f08abf14a791d8ecb7fdab353 docs diff -r 06842841bf42 -r 27795b18326b notes/20230730.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/notes/20230730.org Fri Nov 24 22:39:07 2023 -0500 @@ -0,0 +1,36 @@ +* VC infrastructure +In heptapod we have a root group named =comp=, containg a variety of +subgroups. Some of these groups should be public, while others are +internal to comp members exclusively. Within each subgroup, we should +have the root group members automatically granted privileged access to +projects. This is relevant for the =startup= subgroup in particular, +where each project is potentially maintained by multiple non-root +contributors. + +We also need to consider how we will manage subrepos across the +organization. It is about time we start integrating HG bundles and +potentially mirrors. For our core VC pipeline we should have no +reliance on Git, but this may be difficult. It depends on the behavior +of HG bundles. + +Bookmarks/tags should be used for milestones in the root group and are +infrequent. They are more frequent in projects with a regular release +life-cycle. +* Approaching Webapps +I started poking around in the webapp space again so that I can launch +a landing page for NAS-T quickly. The Rust situation has improved +somewhat on the frontend side, and the axum backend stack is nice. + +This might seem like a lot of Rust and not a lot of Lisp, which it is, +but there's still room for Lisp wherever we need it. It mostly plays a +role in the backend, servicing the database and responding to requests +from the Rust edges. All of the important tests for the web APIs are +also written in Lisp. We will almost certainly use Lisp for all static +processing and HTML generation at compile-time. + +This I believe, is the appropriate way to integrate Lisp into a +cutting-edge web-app. You get the good parts of Lisp where you need +them (interactive debugging, dynamic language, REPL) and avoid the bad +parts (OOB optimization, RPS performance) in areas where the customer +would be impacted. In this domain, Lisp takes the form of a glue +rather than the bricks and mortar it sometimes appears to us as. diff -r 06842841bf42 -r 27795b18326b notes/20231024.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/notes/20231024.org Fri Nov 24 22:39:07 2023 -0500 @@ -0,0 +1,59 @@ +* virt +** QEMU +** KVM +** Hyper-V +** Firecracker +** Docker +** Vagrant +** LXC +** LXD +** containerd +** systemd-nspawn +** VirtualBox + +* Concatenative +** Factor :factor: +- [2023-07-04 Tue] + Factor is a cool concatenative lang but unfortunately the C interface + (vm/master.h) no longer exists on the master branch. +** Joy :joy: + +*** https://hypercubed.github.io/joy/html/j02maf.html + +*** [[https://builds.openlogicproject.org/content/incompleteness/arithmetization-syntax/arithmetization-syntax.pdf][arithmetization of syntax]] +* Lisp :lisp: +These notes pertain to Lisp. More specifically, ANSI Common Lisp in +most places. + +- https://github.com/lispnik/iup/ - doesn't support MacOS yet, looks + cool though + - what we really need is wasm compiler.. TBD +* Rust +** Serde +- [2023-07-05 Wed] \\ + important part of the Rust ecosystem, another dtolnay + contribution. If you want to program a /data/ format in the Rust + ecosystem, this is how you do it. + + The way it works is that you define some special structs, a + Serializer and a Deserializer which implement the Serialize and + Deserialize traits provided by serde, respectively. + + You can use these structs to provide your public API. The + conventional choice is public top-level functions like from-str + and to-string. That's it, your serialization library can now read and + write your data format as Rust data types. + + [[https://serde.rs/enum-representations.html][enum-representations]] + - the default behavior is an externally tagged representation (verbose) + + The docs use strings as core IO when implementing a custom format, + but the convention is to implement for T where T is bound by std::io + Read or Write trait. Then you can provide a more robust public API + (from_bytes, from_writer, etc). +* C +* CPP +* Nu +[[https://www.nushell.sh/][~]] +[[https://www.nushell.sh/cookbook/][cookbook]] +[[https://github.com/nushell/nu_scripts][nu_scripts]] diff -r 06842841bf42 -r 27795b18326b notes/20231101.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/notes/20231101.org Fri Nov 24 22:39:07 2023 -0500 @@ -0,0 +1,18 @@ +* AWS usage +We're leveraging AWS for some of our public web servers for now. It's +really not realistic to expect that my home desktop and spotty Comcast +internet can serve any production workflow. What it /is/ capable of is +a private VPN, which can communicate with AWS and other cloud VPN +depots via WireGuard ([[https://dev.to/gabrieltetzner/setting-up-a-vpn-with-wireguard-server-on-aws-ec2-4a49][article]]). + +I currently use Google Domains for nas-t.net, otom8.dev, and +rwest.io - but that business is now owned by squarespace, so I would +rather move it to Route53. + +We have archlinux ec2 image builds [[https://wiki.archlinux.org/title/Arch_Linux_AMIs_for_Amazon_Web_Services][here]] and [[https://gitlab.com/anemos-io/archlinux-ec2][here]] - only half work and not +maintained, but it's a start. I'm not even sure if I should stick with +arch or cave and use Ubuntu or AWS Linux. We can serve the static +services with little cost, the only big spender will be the heptapod +instance which requires a larger instance and some workers. + +We'll try to keep the cost at or around $30/month. diff -r 06842841bf42 -r 27795b18326b notes/20231102.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/notes/20231102.org Fri Nov 24 22:39:07 2023 -0500 @@ -0,0 +1,82 @@ +* IDEAS +** TODO shed +:PROPERTIES: +:ID: fc9a94e1-91c5-4915-90b8-73218fa3b8bc +:END: +:LOGBOOK: +- State "TODO" from [2023-04-07 Fri 23:24] +:END: +rlib +> ulib +> ulib +> ulib +> ulib + +*** TODO sh* tools +:PROPERTIES: +:ID: c0613a13-7ccb-4af9-b47e-e14a41c782c2 +:END: +:LOGBOOK: +- State "TODO" from "TODO" [2023-04-07 Fri 23:22] +:END: +shc,shx,etc +** WIP packy +:LOGBOOK: +- State "TODO" from [2023-04-07 Fri 23:33] +:END: +*** WIP rust +*** WIP common-lisp +*** WIP emacs-lisp +*** python +*** julia +*** C +*** C++ +** TODO tenex +:LOGBOOK: +- State "TODO" from [2023-04-07 Fri 23:52] +:END: +** TODO mpk +:LOGBOOK: +- State "TODO" from [2023-04-07 Fri 23:52] +:END: +** TODO cfg +:LOGBOOK: +- State "TODO" from [2023-04-07 Fri 23:34] +:END: +** TODO obj +:LOGBOOK: +- State "TODO" from [2023-04-07 Fri 23:51] +:END: +split out from rlib to separate package +- a purely OOP class library +** TODO lab +:LOGBOOK: +- State "TODO" from [2023-04-07 Fri 23:34] +:END: +** TODO source categories +- need a way of extracting metadata from a repo +- need ability to search and query libs/packages +- separate modules based on where they belong in our stack? + - app + - lib + - script? + - dist + - software distros +** TODO generic query language +from obj protocol? +sql compatibility? + +/check out kdb/ +** TODO bbdb +:LOGBOOK: +- Note taken on [2023-10-24 Tue 22:16] \\ + graph database, build on rocksdb +:END: +insidious Big Brother database. +- an application built with obj +- sql + +** TODO NAS-TV :nas:t: +- media streaming +- gstreamer backend +- audio/video diff -r 06842841bf42 -r 27795b18326b notes/20231105.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/notes/20231105.org Fri Nov 24 22:39:07 2023 -0500 @@ -0,0 +1,136 @@ +* DRAFT dylib-skel-1 +- State "DRAFT" from [2023-11-05 Sun 22:23] +** Overview +Our core languages are [[https://www.rust-lang.org/][Rust]] and [[https://lisp-lang.org/][Lisp]] - this is the killer combo which will allow NAS-T +to rapidly develop high-quality software. As such, it's crucial that these two very +different languages (i.e. compilers) are able to interoperate seamlessly. + +Some interop methods are easy to accomodate via the OS - such as IPC or data sharing, +but others are a bit more difficult. + +In this 2-part series we'll build a FFI bridge between Rust and Lisp, which is something +that /can/ be difficult, due to some complications with Rust and because this is not the +most popular software stack (yet ;). This is an experiment and may not make it to our +code-base, but it's definitely something worth adding to the toolbox in case we need it. + +** FFI +The level of interop we're after in this case is [[https://en.wikipedia.org/wiki/Foreign_function_interface][FFI]]. + +Basically, calling Rust code from Lisp and vice-versa. There's an article about calling +Rust from Common Lisp [[https://dev.to/veer66/calling-rust-from-common-lisp-45c5][here]] which shows the basics and serves as a great starting point +for those interested. +*** Rust != C +The complication(s) with Rust I mentioned early is really just that /it is not C/. =C= +is old, i.e. well-supported with a stable ABI, making the process of creating bindings +for a C library a breeze in many languages. + +For a Rust library we need to first appease the compiler, as explained in [[https://doc.rust-lang.org/nomicon/ffi.html#calling-rust-code-from-c][this section]] +of the Rustonomicon. Among other things it involves changing the calling-convention of +functions with a type signature and editing the Cargo.toml file to produce a +C-compatible ABI binary. The Rust default ABI is unstable and can't reliably be used +like the C ABI can. + +*** Overhead +Using FFI involves some overhead. Check [[https://github.com/dyu/ffi-overhead][here]] for an example benchmark across a few +languages. While building the NAS-T core, I'm very much aware of this, and will need a +few sanity benchmarks to make sure the cost doesn't outweigh the benefit. In particular, +I'm concerned about crossing multiple language barriers (Rust<->C<->Lisp). + +** Rust -> C -> Lisp +*** Setup +For starters, I'm going to assume we all have Rust (via =rustup=) and Lisp (=sbcl= only) +installed on our GNU/Linux system (some tweaks needed for Darwin/Windows, not covered in +this post). +**** Cargo +Create a new library crate. For this example we're focusing on a 'skeleton' for +/dynamic/ libraries only, so our experiment will be called =dylib-skel= or *dysk* for +short. +src_sh[:exports code]{cargo init dysk --lib && cd dysk} + +A =src/lib.rs= will be generated for you. Go ahead and delete that. We're going to be +making our own =lib.rs= file directly in the root directory (just to be cool). + +The next step is to edit your =Cargo.toml= file. Add these lines after the =[package]= +section and before =[dependencies]=: +#+begin_src conf-toml +[lib] +crate-type = ["cdylib","rlib"] +path = "lib.rs" +[[bin]] +name="dysk-test" +path="test.rs" +#+end_src + +This tells Rust to generate a shared C-compatible object with a =.so= extension which we +can open using [[https://man.archlinux.org/man/dlopen.3.en][dlopen]]. +**** cbindgen +***** install +Next, we want the =cbindgen= program which we'll use to generate header files for +C/C++. This step isn't necessary at all, we just want it for further experimentation. + +src_sh[:exports code]{cargo install --force cbindgen} + +We append the =cbindgen= crate as a /build dependency/ to our =Cargo.toml= like so: +#+begin_src conf-toml +[build-dependencies] +cbindgen = "0.24" +#+end_src +***** cbindgen.toml +#+begin_src conf-toml :tangle cbindgen.toml +language = "C" +autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */" +include_version = true +namespace = "dysk" +cpp_compat = true +after_includes = "#define DYSK_VERSION \"0.1.0\"" +line_length = 88 +tab_width = 2 +documentation = true +documentation_style = "c99" +usize_is_size_t = true +[cython] +header = '"dysk.h"' +#+end_src +***** build.rs +#+begin_src rust :tangle build.rs +fn main() -> Result<(), cbindgen::Error> { + if let Ok(b) = cbindgen::generate(std::env::var("CARGO_MANIFEST_DIR").unwrap()) { + b.write_to_file("dysk.h"); Ok(())} + else { panic!("failed to generate dysk.h from cbindgen.toml") } } +#+end_src +*** lib.rs +#+begin_src rust :tangle lib.rs +//! lib.rs --- dysk library +use std::ffi::{c_char, c_int, CString}; +#[no_mangle] +pub extern "C" fn dysk_hello() -> *const c_char { + CString::new("hello from rust").unwrap().into_raw()} +#[no_mangle] +pub extern "C" fn dysk_plus(a:c_int,b:c_int) -> c_int {a+b} +#[no_mangle] +pub extern "C" fn dysk_plus1(n:c_int) -> c_int {n+1} +#+end_src +*** test.rs +#+begin_src rust :tangle test.rs +//! test.rs --- dysk test +fn main() { let mut i = 0u32; while i < 500000000 {i+=1; dysk::dysk_plus1(2 as core::ffi::c_int);}} +#+end_src +*** compile +#+begin_src sh +cargo build --release +#+end_src +*** load from SBCL +#+begin_src lisp :tangle dysk.lisp +(load-shared-object #P"target/release/libdysk.so") +(define-alien-routine dysk-hello c-string) +(define-alien-routine dysk-plus int (a int) (b int)) +(define-alien-routine dysk-plus1 int (n int)) +(dysk-hello) ;; => "hello from rust" +#+end_src +*** benchmark +#+begin_src shell +time target/release/dysk-test +#+end_src +#+begin_src lisp :tangle test.lisp +(time (dotimes (_ 500000000) (dysk-plus1 2))) +#+end_src diff -r 06842841bf42 -r 27795b18326b notes/20231124.org --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/notes/20231124.org Fri Nov 24 22:39:07 2023 -0500 @@ -0,0 +1,33 @@ +* cl-dot examples +#+begin_src lisp +(defmethod cl-dot:graph-object-node ((graph (eql 'example)) (object cons)) + (make-instance 'cl-dot:node + :attributes '(:label "cell \\N" + :shape :box))) +(defmethod cl-dot:graph-object-points-to ((graph (eql 'example)) (object cons)) + (list (car object) + (make-instance 'cl-dot:attributed + :object (cdr object) + :attributes '(:weight 3)))) +;; Symbols +(defmethod cl-dot:graph-object-node ((graph (eql 'example)) (object symbol)) + (make-instance 'cl-dot:node + :attributes `(:label ,object + :shape :hexagon + :style :filled + :color :black + :fillcolor "#ccccff"))) +(let* ((data '(a b c #1=(b z) c d #1#)) + (dgraph (cl-dot:generate-graph-from-roots 'example (list data) + '(:rankdir "LR" :layout "twopi" :labelloc "t")))) + (cl-dot:dot-graph dgraph "test-lr.svg" :format #+nil :x11 :svg)) +#+end_src + +#+RESULTS: + +#+begin_src lisp +(let* ((data '(a b)) + (dgraph (cl-dot:generate-graph-from-roots 'example (list data) + '(:rankdir "LR")))) + (cl-dot:print-graph dgraph)) +#+end_src diff -r 06842841bf42 -r 27795b18326b notes/refs.bib --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/notes/refs.bib Fri Nov 24 22:39:07 2023 -0500 @@ -0,0 +1,169 @@ +@article{btrfs, +author = {Rodeh, Ohad and Bacik, Josef and Mason, Chris}, +year = {2013}, +month = {08}, +pages = {}, +title = {BTRFS: The linux B-tree filesystem}, +volume = {9}, +journal = {ACM Transactions on Storage (TOS)}, +doi = {10.1145/2501620.2501623} +} +@INPROCEEDINGS{zfs, + author={Rodeh, O. and Teperman, A.}, + booktitle={20th IEEE/11th NASA Goddard Conference on Mass Storage Systems and Technologies, 2003. (MSST 2003). Proceedings.}, + title={zFS - a scalable distributed file system using object disks}, + year={2003}, + volume={}, + number={}, + pages={207-218}, + doi={10.1109/MASS.2003.1194858}} + +@inproceedings{tmpfs, + title={tmpfs: A Virtual Memory File System}, + author={Peter Snyder}, + year={1990}, + url={https://api.semanticscholar.org/CorpusID:54156693} +} + +@Article{nvme-ssd-ux, +AUTHOR = {Kim, Seongmin and Kim, Kyusik and Shin, Heeyoung and Kim, Taeseok}, +TITLE = {Practical Enhancement of User Experience in NVMe SSDs}, +JOURNAL = {Applied Sciences}, +VOLUME = {10}, +YEAR = {2020}, +NUMBER = {14}, +ARTICLE-NUMBER = {4765}, +URL = {https://www.mdpi.com/2076-3417/10/14/4765}, +ISSN = {2076-3417}, +DOI = {10.3390/app10144765} +} + +@inproceedings{ext4, +author = {Djordjevic, Borislav and Timcenko, Valentina}, +title = {Ext4 File System Performance Analysis in Linux Environment}, +year = {2011}, +isbn = {9781618040282}, +publisher = {World Scientific and Engineering Academy and Society (WSEAS)}, +address = {Stevens Point, Wisconsin, USA}, +booktitle = {Proceedings of the 11th WSEAS International Conference on Applied Informatics and Communications, and Proceedings of the 4th WSEAS International Conference on Biomedical Electronics and Biomedical Informatics, and Proceedings of the International Conference on Computational Engineering in Systems Applications}, +pages = {288–293}, +numpages = {6}, +keywords = {Linux, journaling, ext4/ext3/ext2, filesystems, inodes, disk performances, file block allocation}, +location = {Florence, Italy}, +series = {AIASABEBI'11} +} + +@Article{hd-failure-ml, +AUTHOR = {Zhang, Mingyu and Ge, Wenqiang and Tang, Ruichun and Liu, Peishun}, +TITLE = {Hard Disk Failure Prediction Based on Blending Ensemble Learning}, +JOURNAL = {Applied Sciences}, +VOLUME = {13}, +YEAR = {2023}, +NUMBER = {5}, +ARTICLE-NUMBER = {3288}, +URL = {https://www.mdpi.com/2076-3417/13/5/3288}, +ISSN = {2076-3417}, +DOI = {10.3390/app13053288} +} + +@inproceedings{smart-ssd-qp, +author = {Do, Jaeyoung and Kee, Yang-Suk and Patel, Jignesh M. and Park, Chanik and Park, Kwanghyun and DeWitt, David J.}, +title = {Query Processing on Smart SSDs: Opportunities and Challenges}, +year = {2013}, +isbn = {9781450320375}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +url = {https://doi.org/10.1145/2463676.2465295}, +doi = {10.1145/2463676.2465295}, +booktitle = {Proceedings of the 2013 ACM SIGMOD International Conference on Management of Data}, +pages = {1221–1230}, +numpages = {10}, +keywords = {smart ssd}, +location = {New York, New York, USA}, +series = {SIGMOD '13} +} + +@inproceedings{ssd-perf-opt, +author = {Zuck, Aviad and G\"{u}hring, Philipp and Zhang, Tao and Porter, Donald E. and Tsafrir, Dan}, +title = {Why and How to Increase SSD Performance Transparency}, +year = {2019}, +isbn = {9781450367271}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +url = {https://doi.org/10.1145/3317550.3321430}, +doi = {10.1145/3317550.3321430}, +booktitle = {Proceedings of the Workshop on Hot Topics in Operating Systems}, +pages = {192–200}, +numpages = {9}, +location = {Bertinoro, Italy}, +series = {HotOS '19} +} + +@article{flash-openssd-systems, +author = {Kwak, Jaewook and Lee, Sangjin and Park, Kibin and Jeong, Jinwoo and Song, Yong Ho}, +title = {Cosmos+ OpenSSD: Rapid Prototype for Flash Storage Systems}, +year = {2020}, +issue_date = {August 2020}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +volume = {16}, +number = {3}, +issn = {1553-3077}, +url = {https://doi.org/10.1145/3385073}, +doi = {10.1145/3385073}, +journal = {ACM Trans. Storage}, +month = {jul}, +articleno = {15}, +numpages = {35}, +keywords = {storage system, solid state drive (SSD), flash translation layer (FTL), Flash memory} +} + +@INPROCEEDINGS{emmc-mobile-io, + author={Zhou, Deng and Pan, Wen and Wang, Wei and Xie, Tao}, + booktitle={2015 IEEE International Symposium on Workload Characterization}, + title={I/O Characteristics of Smartphone Applications and Their Implications for eMMC Design}, + year={2015}, + volume={}, + number={}, + pages={12-21}, + doi={10.1109/IISWC.2015.8}} + +@inproceedings{xfs-scalability, + author = {Adam Sweeney and + Doug Doucette and + Wei Hu and + Curtis Anderson and + Mike Nishimoto and + Geoff Peck}, + title = {Scalability in the {XFS} File System}, + booktitle = {Proceedings of the {USENIX} Annual Technical Conference, San Diego, + California, USA, January 22-26, 1996}, + pages = {1--14}, + publisher = {{USENIX} Association}, + year = {1996}, + url = {http://www.usenix.org/publications/library/proceedings/sd96/sweeney.html}, + timestamp = {Wed, 04 Jul 2018 13:06:34 +0200}, + biburl = {https://dblp.org/rec/conf/usenix/Sweeney96.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} + +@inproceedings{xfs, + title={xFS: A wide area mass storage file system}, + author={Wang, Randolph Y and Anderson, Thomas E}, + booktitle={Proceedings of IEEE 4th Workshop on Workstation Operating Systems. WWOS-III}, + pages={71--78}, + year={1993}, + organization={IEEE} +} + +@inproceedings {zns-usenix, +author = {Matias Bj{\o}rling and Abutalib Aghayev and Hans Holmberg and Aravind Ramesh and Damien Le Moal and Gregory R. Ganger and George Amvrosiadis}, +title = {{ZNS}: Avoiding the Block Interface Tax for Flash-based {SSDs}}, +booktitle = {2021 USENIX Annual Technical Conference (USENIX ATC 21)}, +year = {2021}, +isbn = {978-1-939133-23-6}, +pages = {689--703}, +url = {https://www.usenix.org/conference/atc21/presentation/bjorling}, +publisher = {USENIX Association}, +month = jul, +} \ No newline at end of file diff -r 06842841bf42 -r 27795b18326b pitch.org --- a/pitch.org Mon Nov 20 21:13:44 2023 -0500 +++ b/pitch.org Fri Nov 24 22:39:07 2023 -0500 @@ -1,5 +1,4 @@ -#+AUTHOR: Richard Westhaver -#+OPTIONS: ^:nil toc:nil num:nil html-postamble:nil -#+HTML_HEAD: -#+HTML_HEAD: -#+HTML_HEAD: +{{{header(pitch, +Richard Westhaver, +ellis@rwest.io, +The Big Picture)}}} diff -r 06842841bf42 -r 27795b18326b publish.el --- a/publish.el Mon Nov 20 21:13:44 2023 -0500 +++ b/publish.el Fri Nov 24 22:39:07 2023 -0500 @@ -24,7 +24,8 @@ org-html-validation-link nil org-src-fontify-natively t make-backup-files nil - debug-on-error t) + debug-on-error t + org-id-link-to-org-use-id t) (setq org-publish-project-alist `(("compiler.company" :components ("index" "blog" "docs")) diff -r 06842841bf42 -r 27795b18326b readme.org --- a/readme.org Mon Nov 20 21:13:44 2023 -0500 +++ b/readme.org Fri Nov 24 22:39:07 2023 -0500 @@ -11,7 +11,6 @@ - [[file:blog][blog]] - [2023-11-19 Sun] [[https://compiler.company/blog/hello-world][hello-world]] - - [[file:docs][docs]] - [[file:roadmap.org][roadmap]] - [[file:pitch.org][pitch]] diff -r 06842841bf42 -r 27795b18326b roadmap.org --- a/roadmap.org Mon Nov 20 21:13:44 2023 -0500 +++ b/roadmap.org Fri Nov 24 22:39:07 2023 -0500 @@ -1,11 +1,7 @@ -#+TITLE: roadmap -#+AUTHOR: Richard Westhaver -#+EMAIL: ellis@rwest.io -#+OPTIONS: ^:nil toc:nil num:nil -#+HTML_HEAD: -#+HTML_HEAD: -#+HTML_HEAD: - +{{{header(roadmap, +Richard Westhaver, +ellis@rwest.io, +The Compiler Company Roadmap)}}} * WIP 2023 SCHEDULED: <2023-01-01 Mon>--[2023-12-31 Tue] - State "WIP" from [2023-11-05 Sun 21:47] diff -r 06842841bf42 -r 27795b18326b scratch/20230730.org --- a/scratch/20230730.org Mon Nov 20 21:13:44 2023 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,36 +0,0 @@ -* VC infrastructure -In heptapod we have a root group named =comp=, containg a variety of -subgroups. Some of these groups should be public, while others are -internal to comp members exclusively. Within each subgroup, we should -have the root group members automatically granted privileged access to -projects. This is relevant for the =startup= subgroup in particular, -where each project is potentially maintained by multiple non-root -contributors. - -We also need to consider how we will manage subrepos across the -organization. It is about time we start integrating HG bundles and -potentially mirrors. For our core VC pipeline we should have no -reliance on Git, but this may be difficult. It depends on the behavior -of HG bundles. - -Bookmarks/tags should be used for milestones in the root group and are -infrequent. They are more frequent in projects with a regular release -life-cycle. -* Approaching Webapps -I started poking around in the webapp space again so that I can launch -a landing page for NAS-T quickly. The Rust situation has improved -somewhat on the frontend side, and the axum backend stack is nice. - -This might seem like a lot of Rust and not a lot of Lisp, which it is, -but there's still room for Lisp wherever we need it. It mostly plays a -role in the backend, servicing the database and responding to requests -from the Rust edges. All of the important tests for the web APIs are -also written in Lisp. We will almost certainly use Lisp for all static -processing and HTML generation at compile-time. - -This I believe, is the appropriate way to integrate Lisp into a -cutting-edge web-app. You get the good parts of Lisp where you need -them (interactive debugging, dynamic language, REPL) and avoid the bad -parts (OOB optimization, RPS performance) in areas where the customer -would be impacted. In this domain, Lisp takes the form of a glue -rather than the bricks and mortar it sometimes appears to us as. diff -r 06842841bf42 -r 27795b18326b scratch/20231024.org --- a/scratch/20231024.org Mon Nov 20 21:13:44 2023 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,59 +0,0 @@ -* virt -** QEMU -** KVM -** Hyper-V -** Firecracker -** Docker -** Vagrant -** LXC -** LXD -** containerd -** systemd-nspawn -** VirtualBox - -* Concatenative -** Factor :factor: -- [2023-07-04 Tue] - Factor is a cool concatenative lang but unfortunately the C interface - (vm/master.h) no longer exists on the master branch. -** Joy :joy: - -*** https://hypercubed.github.io/joy/html/j02maf.html - -*** [[https://builds.openlogicproject.org/content/incompleteness/arithmetization-syntax/arithmetization-syntax.pdf][arithmetization of syntax]] -* Lisp :lisp: -These notes pertain to Lisp. More specifically, ANSI Common Lisp in -most places. - -- https://github.com/lispnik/iup/ - doesn't support MacOS yet, looks - cool though - - what we really need is wasm compiler.. TBD -* Rust -** Serde -- [2023-07-05 Wed] \\ - important part of the Rust ecosystem, another dtolnay - contribution. If you want to program a /data/ format in the Rust - ecosystem, this is how you do it. - - The way it works is that you define some special structs, a - Serializer and a Deserializer which implement the Serialize and - Deserialize traits provided by serde, respectively. - - You can use these structs to provide your public API. The - conventional choice is public top-level functions like from-str - and to-string. That's it, your serialization library can now read and - write your data format as Rust data types. - - [[https://serde.rs/enum-representations.html][enum-representations]] - - the default behavior is an externally tagged representation (verbose) - - The docs use strings as core IO when implementing a custom format, - but the convention is to implement for T where T is bound by std::io - Read or Write trait. Then you can provide a more robust public API - (from_bytes, from_writer, etc). -* C -* CPP -* Nu -[[https://www.nushell.sh/][~]] -[[https://www.nushell.sh/cookbook/][cookbook]] -[[https://github.com/nushell/nu_scripts][nu_scripts]] diff -r 06842841bf42 -r 27795b18326b scratch/20231101.org --- a/scratch/20231101.org Mon Nov 20 21:13:44 2023 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,18 +0,0 @@ -* AWS usage -We're leveraging AWS for some of our public web servers for now. It's -really not realistic to expect that my home desktop and spotty Comcast -internet can serve any production workflow. What it /is/ capable of is -a private VPN, which can communicate with AWS and other cloud VPN -depots via WireGuard ([[https://dev.to/gabrieltetzner/setting-up-a-vpn-with-wireguard-server-on-aws-ec2-4a49][article]]). - -I currently use Google Domains for nas-t.net, otom8.dev, and -rwest.io - but that business is now owned by squarespace, so I would -rather move it to Route53. - -We have archlinux ec2 image builds [[https://wiki.archlinux.org/title/Arch_Linux_AMIs_for_Amazon_Web_Services][here]] and [[https://gitlab.com/anemos-io/archlinux-ec2][here]] - only half work and not -maintained, but it's a start. I'm not even sure if I should stick with -arch or cave and use Ubuntu or AWS Linux. We can serve the static -services with little cost, the only big spender will be the heptapod -instance which requires a larger instance and some workers. - -We'll try to keep the cost at or around $30/month. diff -r 06842841bf42 -r 27795b18326b scratch/20231102.org --- a/scratch/20231102.org Mon Nov 20 21:13:44 2023 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,82 +0,0 @@ -* IDEAS -** TODO shed -:PROPERTIES: -:ID: fc9a94e1-91c5-4915-90b8-73218fa3b8bc -:END: -:LOGBOOK: -- State "TODO" from [2023-04-07 Fri 23:24] -:END: -rlib -> ulib -> ulib -> ulib -> ulib - -*** TODO sh* tools -:PROPERTIES: -:ID: c0613a13-7ccb-4af9-b47e-e14a41c782c2 -:END: -:LOGBOOK: -- State "TODO" from "TODO" [2023-04-07 Fri 23:22] -:END: -shc,shx,etc -** WIP packy -:LOGBOOK: -- State "TODO" from [2023-04-07 Fri 23:33] -:END: -*** WIP rust -*** WIP common-lisp -*** WIP emacs-lisp -*** python -*** julia -*** C -*** C++ -** TODO tenex -:LOGBOOK: -- State "TODO" from [2023-04-07 Fri 23:52] -:END: -** TODO mpk -:LOGBOOK: -- State "TODO" from [2023-04-07 Fri 23:52] -:END: -** TODO cfg -:LOGBOOK: -- State "TODO" from [2023-04-07 Fri 23:34] -:END: -** TODO obj -:LOGBOOK: -- State "TODO" from [2023-04-07 Fri 23:51] -:END: -split out from rlib to separate package -- a purely OOP class library -** TODO lab -:LOGBOOK: -- State "TODO" from [2023-04-07 Fri 23:34] -:END: -** TODO source categories -- need a way of extracting metadata from a repo -- need ability to search and query libs/packages -- separate modules based on where they belong in our stack? - - app - - lib - - script? - - dist - - software distros -** TODO generic query language -from obj protocol? -sql compatibility? - -/check out kdb/ -** TODO bbdb -:LOGBOOK: -- Note taken on [2023-10-24 Tue 22:16] \\ - graph database, build on rocksdb -:END: -insidious Big Brother database. -- an application built with obj -- sql - -** TODO NAS-TV :nas:t: -- media streaming -- gstreamer backend -- audio/video diff -r 06842841bf42 -r 27795b18326b scratch/20231105.org --- a/scratch/20231105.org Mon Nov 20 21:13:44 2023 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,136 +0,0 @@ -* DRAFT dylib-skel-1 -- State "DRAFT" from [2023-11-05 Sun 22:23] -** Overview -Our core languages are [[https://www.rust-lang.org/][Rust]] and [[https://lisp-lang.org/][Lisp]] - this is the killer combo which will allow NAS-T -to rapidly develop high-quality software. As such, it's crucial that these two very -different languages (i.e. compilers) are able to interoperate seamlessly. - -Some interop methods are easy to accomodate via the OS - such as IPC or data sharing, -but others are a bit more difficult. - -In this 2-part series we'll build a FFI bridge between Rust and Lisp, which is something -that /can/ be difficult, due to some complications with Rust and because this is not the -most popular software stack (yet ;). This is an experiment and may not make it to our -code-base, but it's definitely something worth adding to the toolbox in case we need it. - -** FFI -The level of interop we're after in this case is [[https://en.wikipedia.org/wiki/Foreign_function_interface][FFI]]. - -Basically, calling Rust code from Lisp and vice-versa. There's an article about calling -Rust from Common Lisp [[https://dev.to/veer66/calling-rust-from-common-lisp-45c5][here]] which shows the basics and serves as a great starting point -for those interested. -*** Rust != C -The complication(s) with Rust I mentioned early is really just that /it is not C/. =C= -is old, i.e. well-supported with a stable ABI, making the process of creating bindings -for a C library a breeze in many languages. - -For a Rust library we need to first appease the compiler, as explained in [[https://doc.rust-lang.org/nomicon/ffi.html#calling-rust-code-from-c][this section]] -of the Rustonomicon. Among other things it involves changing the calling-convention of -functions with a type signature and editing the Cargo.toml file to produce a -C-compatible ABI binary. The Rust default ABI is unstable and can't reliably be used -like the C ABI can. - -*** Overhead -Using FFI involves some overhead. Check [[https://github.com/dyu/ffi-overhead][here]] for an example benchmark across a few -languages. While building the NAS-T core, I'm very much aware of this, and will need a -few sanity benchmarks to make sure the cost doesn't outweigh the benefit. In particular, -I'm concerned about crossing multiple language barriers (Rust<->C<->Lisp). - -** Rust -> C -> Lisp -*** Setup -For starters, I'm going to assume we all have Rust (via =rustup=) and Lisp (=sbcl= only) -installed on our GNU/Linux system (some tweaks needed for Darwin/Windows, not covered in -this post). -**** Cargo -Create a new library crate. For this example we're focusing on a 'skeleton' for -/dynamic/ libraries only, so our experiment will be called =dylib-skel= or *dysk* for -short. -src_sh[:exports code]{cargo init dysk --lib && cd dysk} - -A =src/lib.rs= will be generated for you. Go ahead and delete that. We're going to be -making our own =lib.rs= file directly in the root directory (just to be cool). - -The next step is to edit your =Cargo.toml= file. Add these lines after the =[package]= -section and before =[dependencies]=: -#+begin_src conf-toml -[lib] -crate-type = ["cdylib","rlib"] -path = "lib.rs" -[[bin]] -name="dysk-test" -path="test.rs" -#+end_src - -This tells Rust to generate a shared C-compatible object with a =.so= extension which we -can open using [[https://man.archlinux.org/man/dlopen.3.en][dlopen]]. -**** cbindgen -***** install -Next, we want the =cbindgen= program which we'll use to generate header files for -C/C++. This step isn't necessary at all, we just want it for further experimentation. - -src_sh[:exports code]{cargo install --force cbindgen} - -We append the =cbindgen= crate as a /build dependency/ to our =Cargo.toml= like so: -#+begin_src conf-toml -[build-dependencies] -cbindgen = "0.24" -#+end_src -***** cbindgen.toml -#+begin_src conf-toml :tangle cbindgen.toml -language = "C" -autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */" -include_version = true -namespace = "dysk" -cpp_compat = true -after_includes = "#define DYSK_VERSION \"0.1.0\"" -line_length = 88 -tab_width = 2 -documentation = true -documentation_style = "c99" -usize_is_size_t = true -[cython] -header = '"dysk.h"' -#+end_src -***** build.rs -#+begin_src rust :tangle build.rs -fn main() -> Result<(), cbindgen::Error> { - if let Ok(b) = cbindgen::generate(std::env::var("CARGO_MANIFEST_DIR").unwrap()) { - b.write_to_file("dysk.h"); Ok(())} - else { panic!("failed to generate dysk.h from cbindgen.toml") } } -#+end_src -*** lib.rs -#+begin_src rust :tangle lib.rs -//! lib.rs --- dysk library -use std::ffi::{c_char, c_int, CString}; -#[no_mangle] -pub extern "C" fn dysk_hello() -> *const c_char { - CString::new("hello from rust").unwrap().into_raw()} -#[no_mangle] -pub extern "C" fn dysk_plus(a:c_int,b:c_int) -> c_int {a+b} -#[no_mangle] -pub extern "C" fn dysk_plus1(n:c_int) -> c_int {n+1} -#+end_src -*** test.rs -#+begin_src rust :tangle test.rs -//! test.rs --- dysk test -fn main() { let mut i = 0u32; while i < 500000000 {i+=1; dysk::dysk_plus1(2 as core::ffi::c_int);}} -#+end_src -*** compile -#+begin_src sh -cargo build --release -#+end_src -*** load from SBCL -#+begin_src lisp :tangle dysk.lisp -(load-shared-object #P"target/release/libdysk.so") -(define-alien-routine dysk-hello c-string) -(define-alien-routine dysk-plus int (a int) (b int)) -(define-alien-routine dysk-plus1 int (n int)) -(dysk-hello) ;; => "hello from rust" -#+end_src -*** benchmark -#+begin_src shell -time target/release/dysk-test -#+end_src -#+begin_src lisp :tangle test.lisp -(time (dotimes (_ 500000000) (dysk-plus1 2))) -#+end_src diff -r 06842841bf42 -r 27795b18326b scratch/refs.bib --- a/scratch/refs.bib Mon Nov 20 21:13:44 2023 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,169 +0,0 @@ -@article{btrfs, -author = {Rodeh, Ohad and Bacik, Josef and Mason, Chris}, -year = {2013}, -month = {08}, -pages = {}, -title = {BTRFS: The linux B-tree filesystem}, -volume = {9}, -journal = {ACM Transactions on Storage (TOS)}, -doi = {10.1145/2501620.2501623} -} -@INPROCEEDINGS{zfs, - author={Rodeh, O. and Teperman, A.}, - booktitle={20th IEEE/11th NASA Goddard Conference on Mass Storage Systems and Technologies, 2003. (MSST 2003). Proceedings.}, - title={zFS - a scalable distributed file system using object disks}, - year={2003}, - volume={}, - number={}, - pages={207-218}, - doi={10.1109/MASS.2003.1194858}} - -@inproceedings{tmpfs, - title={tmpfs: A Virtual Memory File System}, - author={Peter Snyder}, - year={1990}, - url={https://api.semanticscholar.org/CorpusID:54156693} -} - -@Article{nvme-ssd-ux, -AUTHOR = {Kim, Seongmin and Kim, Kyusik and Shin, Heeyoung and Kim, Taeseok}, -TITLE = {Practical Enhancement of User Experience in NVMe SSDs}, -JOURNAL = {Applied Sciences}, -VOLUME = {10}, -YEAR = {2020}, -NUMBER = {14}, -ARTICLE-NUMBER = {4765}, -URL = {https://www.mdpi.com/2076-3417/10/14/4765}, -ISSN = {2076-3417}, -DOI = {10.3390/app10144765} -} - -@inproceedings{ext4, -author = {Djordjevic, Borislav and Timcenko, Valentina}, -title = {Ext4 File System Performance Analysis in Linux Environment}, -year = {2011}, -isbn = {9781618040282}, -publisher = {World Scientific and Engineering Academy and Society (WSEAS)}, -address = {Stevens Point, Wisconsin, USA}, -booktitle = {Proceedings of the 11th WSEAS International Conference on Applied Informatics and Communications, and Proceedings of the 4th WSEAS International Conference on Biomedical Electronics and Biomedical Informatics, and Proceedings of the International Conference on Computational Engineering in Systems Applications}, -pages = {288–293}, -numpages = {6}, -keywords = {Linux, journaling, ext4/ext3/ext2, filesystems, inodes, disk performances, file block allocation}, -location = {Florence, Italy}, -series = {AIASABEBI'11} -} - -@Article{hd-failure-ml, -AUTHOR = {Zhang, Mingyu and Ge, Wenqiang and Tang, Ruichun and Liu, Peishun}, -TITLE = {Hard Disk Failure Prediction Based on Blending Ensemble Learning}, -JOURNAL = {Applied Sciences}, -VOLUME = {13}, -YEAR = {2023}, -NUMBER = {5}, -ARTICLE-NUMBER = {3288}, -URL = {https://www.mdpi.com/2076-3417/13/5/3288}, -ISSN = {2076-3417}, -DOI = {10.3390/app13053288} -} - -@inproceedings{smart-ssd-qp, -author = {Do, Jaeyoung and Kee, Yang-Suk and Patel, Jignesh M. and Park, Chanik and Park, Kwanghyun and DeWitt, David J.}, -title = {Query Processing on Smart SSDs: Opportunities and Challenges}, -year = {2013}, -isbn = {9781450320375}, -publisher = {Association for Computing Machinery}, -address = {New York, NY, USA}, -url = {https://doi.org/10.1145/2463676.2465295}, -doi = {10.1145/2463676.2465295}, -booktitle = {Proceedings of the 2013 ACM SIGMOD International Conference on Management of Data}, -pages = {1221–1230}, -numpages = {10}, -keywords = {smart ssd}, -location = {New York, New York, USA}, -series = {SIGMOD '13} -} - -@inproceedings{ssd-perf-opt, -author = {Zuck, Aviad and G\"{u}hring, Philipp and Zhang, Tao and Porter, Donald E. and Tsafrir, Dan}, -title = {Why and How to Increase SSD Performance Transparency}, -year = {2019}, -isbn = {9781450367271}, -publisher = {Association for Computing Machinery}, -address = {New York, NY, USA}, -url = {https://doi.org/10.1145/3317550.3321430}, -doi = {10.1145/3317550.3321430}, -booktitle = {Proceedings of the Workshop on Hot Topics in Operating Systems}, -pages = {192–200}, -numpages = {9}, -location = {Bertinoro, Italy}, -series = {HotOS '19} -} - -@article{flash-openssd-systems, -author = {Kwak, Jaewook and Lee, Sangjin and Park, Kibin and Jeong, Jinwoo and Song, Yong Ho}, -title = {Cosmos+ OpenSSD: Rapid Prototype for Flash Storage Systems}, -year = {2020}, -issue_date = {August 2020}, -publisher = {Association for Computing Machinery}, -address = {New York, NY, USA}, -volume = {16}, -number = {3}, -issn = {1553-3077}, -url = {https://doi.org/10.1145/3385073}, -doi = {10.1145/3385073}, -journal = {ACM Trans. Storage}, -month = {jul}, -articleno = {15}, -numpages = {35}, -keywords = {storage system, solid state drive (SSD), flash translation layer (FTL), Flash memory} -} - -@INPROCEEDINGS{emmc-mobile-io, - author={Zhou, Deng and Pan, Wen and Wang, Wei and Xie, Tao}, - booktitle={2015 IEEE International Symposium on Workload Characterization}, - title={I/O Characteristics of Smartphone Applications and Their Implications for eMMC Design}, - year={2015}, - volume={}, - number={}, - pages={12-21}, - doi={10.1109/IISWC.2015.8}} - -@inproceedings{xfs-scalability, - author = {Adam Sweeney and - Doug Doucette and - Wei Hu and - Curtis Anderson and - Mike Nishimoto and - Geoff Peck}, - title = {Scalability in the {XFS} File System}, - booktitle = {Proceedings of the {USENIX} Annual Technical Conference, San Diego, - California, USA, January 22-26, 1996}, - pages = {1--14}, - publisher = {{USENIX} Association}, - year = {1996}, - url = {http://www.usenix.org/publications/library/proceedings/sd96/sweeney.html}, - timestamp = {Wed, 04 Jul 2018 13:06:34 +0200}, - biburl = {https://dblp.org/rec/conf/usenix/Sweeney96.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} - -@inproceedings{xfs, - title={xFS: A wide area mass storage file system}, - author={Wang, Randolph Y and Anderson, Thomas E}, - booktitle={Proceedings of IEEE 4th Workshop on Workstation Operating Systems. WWOS-III}, - pages={71--78}, - year={1993}, - organization={IEEE} -} - -@inproceedings {zns-usenix, -author = {Matias Bj{\o}rling and Abutalib Aghayev and Hans Holmberg and Aravind Ramesh and Damien Le Moal and Gregory R. Ganger and George Amvrosiadis}, -title = {{ZNS}: Avoiding the Block Interface Tax for Flash-based {SSDs}}, -booktitle = {2021 USENIX Annual Technical Conference (USENIX ATC 21)}, -year = {2021}, -isbn = {978-1-939133-23-6}, -pages = {689--703}, -url = {https://www.usenix.org/conference/atc21/presentation/bjorling}, -publisher = {USENIX Association}, -month = jul, -} \ No newline at end of file